Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
edge0 initial commit: MLX streaming MoE inference (prerouter + LoRA + SSD offload)
linyubupa17021c587146
diff --git a/.gitignore b/.gitignorenew file mode 100644index 0000000..e1bc0b7--- /dev/null+++ b/.gitignore@@ -0,0 +1,20 @@+# Python+__pycache__/+*.py[cod]+*.egg-info/+build/+dist/+.pytest_cache/+.coverage++# Virtualenvs+.venv/++# Trained adapter artifacts (never committed; see README)+artifacts/++# macOS+.DS_Store++# Bench / runtime logs+*.logdiff --git a/LICENSE b/LICENSEnew file mode 100644index 0000000..d645695--- /dev/null+++ b/LICENSE@@ -0,0 +1,202 @@++ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.diff --git a/NOTICE b/NOTICEnew file mode 100644index 0000000..ddee8a9--- /dev/null+++ b/NOTICE@@ -0,0 +1,21 @@+edge0+=====++This project bundles or adapts the following third-party works:++mlx-lm (MIT License)+ - `src/edge0/backends/mlx/_impl/qwen3_next.py` is vendored from mlx-lm+ 0.31.0 (https://github.com/ml-explore/mlx-lm), with additive hooks for+ prerouter routing and layer callbacks. Non-prerouter paths are kept+ byte-identical to the upstream file.+ - mlx-lm is used at runtime for checkpoint / tokenizer loading.+ Copyright (c) 2023-2024 Apple Inc. and the mlx-lm contributors.++bailing_hybrid (Apache License 2.0)+ - `src/edge0/backends/mlx/_impl/bailing_hybrid.py` is vendored from the+ Ling 3.0 MLX implementation (rapid-mlx / Ling-3.0-tiny, Apache-2.0), with+ prerouter hook identifiers renamed for consistency with the framework.+ Copyright (c) the bailing / Ling MLX contributors.++All other files are Copyright (c) 2025 edge0 contributors, licensed under+the Apache License, Version 2.0 (see LICENSE).diff --git a/README.md b/README.mdnew file mode 100644index 0000000..c74f2e9--- /dev/null+++ b/README.md@@ -0,0 +1,72 @@+# edge0++**edge0** 是一个面向 Apple Silicon 的开源流式 MoE 推理框架:把「SSD 专家 offload ++并行 LoRA + prerouter 路由预判」方案抽象成可扩展的通用框架,开箱支持两个模型:++| 模型 | 说明 | 推理档 |+|---|---|---|+| `edge0-35b` | Qwen3.6-35B-A3B 4bit(40 层,256 专家) | prerouter K=4 |+| `edge0-10b` | Ling-10B bailing_hybrid(24 层,128 专家) | prerouter K=8 |++设计目标:++- **像 transformers 一样使用**:`AutoModel` / `AutoConfig` / `AutoEngine` 按模型名自动选类;+- **后端隔离**:全部 MLX 代码收在 `edge0/backends/mlx/`,核心逻辑只依赖后端门面,+ 为未来 CUDA 后端预留平级插槽(`backends/cuda/`);+- **适配器统一为 safetensors**:LoRA 与 prerouter 权重均为带元数据的 `.safetensors`;+- **旧术语零残留**:预路由头在代码与文档中一律称 `prerouter`。++## 快速开始++```bash+# 1) 安装(Python ≥3.10,Apple Silicon + mlx)+python3.12 -m venv .venv && .venv/bin/pip install -e '.[dev]'++# 2) 一键试玩:自动探测开发机上的 checkpoint;也可以显式传路径或 tier 名+edge0 demo+edge0 demo /path/to/qwen35/model+edge0 demo edge0-10b++# 3) vLLM 风格起服务:模型是位置参数,checkpoint 类型自动识别+edge0 serve /path/to/qwen35/model+edge0 serve edge0-35b # tier 名:EDGE0_35B_MODEL 环境变量可覆盖默认路径++curl http://127.0.0.1:8000/v1/chat/completions \+ -H 'Content-Type: application/json' \+ -d '{"messages":[{"role":"user","content":"Hello!"}],"max_tokens":32}'++# 4) 命令行一问一答(同样接受路径或 tier 名)+edge0 chat /path/to/model --prompt "你好,用一句话介绍珠海。"+```++`python -m edge0 ...` 与 `edge0 ...` 完全等价。++### 模型与适配器++- **checkpoint**:qwen35 / ling v7 的原始模型目录(含 `config.json`、`model.safetensors`、+ `tokenizer.json`)。`edge0 serve <dir>` 按 `config.json` 自动识别 tier。+- **适配器**(LoRA + prerouter,`artifacts/` 目录内自动使用,无需手动指定):+ - `artifacts/lora_edge0_35b_k4.safetensors` + `artifacts/prerouter_edge0_35b_k4.safetensors`+ - `artifacts/lora_edge0_10b.safetensors` + `artifacts/prerouter_edge0_10b.safetensors`+ - 旧版 npz 迁移:`edge0 convert-adapters`++### 验证++```bash+pytest # 单元测试(不含真实权重)+pytest -m slow # 真实 checkpoint 端到端(qwen/ling 生成 + HTTP)+scripts/e2e_smoke.py # 两档模型 staged vs exact 数值一致性冒烟+scripts/zhuhai_example.py # 完整 API 上手例子(--thinking 开关演示)+examples/demo.py # 最小 API walkthrough(edge0 demo 的等价代码)+```++## 文档++- [架构总览](docs/architecture.md)+- [注意力抽象](docs/attention.md) / [MoE 抽象](docs/moe.md) / [SSD 流式](docs/streaming.md) / [prerouter](docs/prerouter.md)+- [如何接入新模型](docs/adding-a-model.md)+- [edge0-35b](docs/models/edge0-35b.md) / [edge0-10b](docs/models/edge0-10b.md)++## License++Apache-2.0,含 vendored 第三方代码(详见 [NOTICE](NOTICE.md))。diff --git a/docs/adding-a-model.md b/docs/adding-a-model.mdnew file mode 100644index 0000000..5e784ec--- /dev/null+++ b/docs/adding-a-model.md@@ -0,0 +1,207 @@+# 接入一个新模型++本文档给出向 edge0 接入一个新模型族的分步指南,用 `edge0-35b`(`src/edge0/models/+edge0_35b/__init__.py`)作为逐步示例,逐处标注真实代码行号。`edge0-10b`+(`src/edge0/models/edge0_10b/__init__.py`)是第二个活例子,路径解析与契约完全一致。++接入目标:让 `AutoConfig` / `AutoModel` / `AutoEngine` 能按模型名(或 checkpoint 的+`config.json` 的 `model_type`)解析你的模型,且流式 MoE 层能由一个通用的+`StreamingSwitchGLU` 驱动。++## 接入总览(契约)++`edge0.registry` 的契约是:**一个适配器模块暴露三件套**:++- `Config` —— 一个 `ModelConfig` 子类(含 `from_pretrained`);+- `build_model(model_dir, **overrides)` —— 装配好的模型骨架;+- `build_engine(model_dir, **overrides)` —— 可生成引擎。++`register_model(name, adapter)` 注册;`TYPE_ALIASES` 把 checkpoint 的 `model_type`+映射到注册名。见 `registry.py` 的 `_resolve_name` 与 `AutoConfig.from_pretrained`+(registry.py:73–101)。++## 第 1 步:创建适配器模块与目录++在 `src/edge0/models/` 下建一个包,名取你的档位,例如 `edge0_35b/`。目录内至少一个+`__init__.py` 作为适配器。模型目录布局要求见「模型目录布局」一节。++## 第 2 步:定义 `Config` 子类与 `_defaults`++`Config` 是 `ModelConfig`(`models/base.py`)的子类。`ModelConfig` 的字段即「跑起一个+模型档位需要的一切」:++| 字段 | 类型 | 语义 |+| --- | --- | --- |+| `name` | `str` | 注册名 |+| `model_dir` | `str` | checkpoint 目录 |+| `moe_spec` | `MoESpec` | MoE 规格(见 moe.md) |+| `options` | `LayerOptions` | 流式层选项 |+| `prerouter` | `PrerouterSpec \| None` | 预路由头规格 |+| `prerouter_top_k` | `int` | 预路由宽度(0 → 用 `options.top_k`) |+| `lora` / `lora_r` / `lora_alpha` | `str` / `int` / `float` | LoRA 权重路径与超参(`""` 禁用) |+| `gen` | `GenerationConfig` | 采样默认(温度 / top-p / top-k / eos 等) |+| `prefill_chunk` / `hot_window` / `intra_staging` / `prefetch_history` | — | 流式与预取行为 |+| `port` | `int` | 服务端口 |+| `target_tok_s` / `peak_active_mem_mb` | `float` | 验收指标(生产机实测) |++子类只需实现类方法 `_defaults(model_dir) -> Config`,返回族默认配置。edge0-35b 的+实现见 `edge0_35b/__init__.py:31–69`:`_defaults` 用一份完整的 `MoESpec`、`LayerOptions`+预设 `staged_k4()`、`PrerouterSpec`、采样默认与验收指标构造 `Qwen35Config`。++`from_pretrained(model_dir=None, **overrides)` 是 `ModelConfig` 提供的模板方法+(`models/base.py:60–73`):它调用 `_defaults` 得到基配置,再对每个 override 校验后+`replace` 覆盖;未知字段抛 `TypeError` 并列出已知字段。因此**每个 public 属性都能被+用户 override**。++要点:++- 仓库根目录有 `scripts/convert_adapters_legacy.py` 一次性把训练 npz 导出转成+ safetensors 产物到 `artifacts/`;`ModelConfig.artifact(name)`(`models/base.py:28–30`)+ 返回这些产物的绝对路径,适配器用它填 LoRA / prerouter 权重路径。+- LoRA override 走 `resolve_lora`(`models/base.py:88–95`):裸模型名解析为该档产物,+ `"model_dir"` 表示「保留训练权重在原位」,空串禁用。++## 第 3 步:`build_model` / `build_engine`++两个模块级函数,复用引擎自己的构建路径(DRY)。++`edge0_35b/__init__.py:72–79`:++```python+def build_model(model_dir=None, **overrides):+ from edge0.engine.qwen import load_installed+ cfg = Qwen35Config.from_pretrained(model_dir, **overrides)+ model, _mcfg, _shards, _installs = load_installed(cfg.model_dir, cfg)+ return model+```++`edge0_35b/__init__.py:82–87`:++```python+def build_engine(model_dir=None, **overrides):+ from edge0.engine.qwen import Qwen35Engine+ cfg = Qwen35Config.from_pretrained(model_dir, **overrides)+ return Qwen35Engine(cfg.model_dir, cfg)+```++你的模型若族数学不同(如 edge0-10b 的 `SIGMOID_GROUP` + 混合 MLA),在+`src/edge0/engine/` 写一个对应引擎(参照 `engine/qwen.py` / `engine/ling.py`),+`build_model` / `build_engine` 引用它。`load_installed` 负责装配流式孪生 / LoRA /+prerouter,是框架给引擎提供的构建路径。++## 第 4 步:注册到注册表++文件末尾调用 `register_model`,并把 `Config` 暴露为模块属性(契约要求):++```python+# edge0_35b/__init__.py:90–93+register_model("edge0-35b", sys.modules[__name__])+Config = Qwen35Config # registry contract: adapter.Config+```++- `register_model`(`registry.py:25–28`):重名抛 `ValueError`。+- 适配器模块必须被 import 才会注册。`edge0.models.__init__`(models/__init__.py:15)+ 显式 import 各档包来 populate 注册表,`AutoConfig.from_pretrained` 里也会+ `from edge0 import models` 兜底。+- 你的模型包应加进 `edge0/models/__init__.py` 的 import 列表。++## 第 5 步:`TYPE_ALIASES`++`TYPE_ALIASES`(`registry.py:15–22`)把 checkpoint `config.json` 的 `model_type`+映射到注册名,使 `AutoEngine.from_pretrained(model_dir=...)` 能**直接从目录解析**而无需+显式传 name。例:++```python+TYPE_ALIASES = {+ "qwen3_5_moe_text": "edge0-35b",+ "qwen3_5_moe": "edge0-35b",+ "bailing_hybrid": "edge0-10b",+ "bailing_moe_linear": "edge0-10b",+}+```++`_resolve_name`(registry.py:42–56)的查找顺序:显式 `name` → 目录的 `model_type`+(`_model_type_from_dir` 读 `config.json`,失败则退回目录 basename)→ 匹配+`MODEL_REGISTRY` → 匹配 `TYPE_ALIASES` → 否则 `KeyError` 列出已注册模型。+未知名会因 `test_unknown_name_lists_registry` 之类的断言被测试钉住。++## 模型目录布局要求++`AutoConfig.from_pretrained(model_dir=...)` 会读 `model_dir/config.json` 的+`model_type` 字段(`registry.py:58–70`)。因此 checkpoint 目录至少要满足:++- `config.json` 存在,且 `model_type` 已登记(或你的 `TYPE_ALIASES` 覆盖该值);+- 权重为 safetensors 格式,key 前缀与 `moe_spec.key_template` 一致(如+ `language_model.model.layers.N.mlp.switch_mlp`);+- LoRA / prerouter 权重为带元数据的 `.safetensors`(`models/base.py` 顶部注释:由+ `convert_adapters_legacy.py` 从训练 npz 一次性转换)。++目录 basename 是最后兜底的解析手段(`_model_type_from_dir`),所以目录名最好与+注册名对齐,但不是必须。++## 第 6 步:测试建议++参考 `tests/test_moe_spec.py` 与 `tests/test_registry.py`(纯逻辑单测,无需真实+checkpoint)。++- **注册与别名**(test_registry.py:12–20):断言 `MODEL_REGISTRY` 含你的注册名、+ `TYPE_ALIASES` 解析正确。+- **Config 默认值**(test_registry.py:23–33):`AutoConfig.from_pretrained(name=...)`+ 的 `moe_spec` / `options` / `prerouter` / `prerouter_top_k` 等字段符合预期。+- **profile 细化**(test_registry.py:35–71):逐字段断言你的档位数值(专家数、top_k、+ quant、端口、验收指标等)。+- **override 与拒绝**(test_registry.py:74–81):`from_pretrained(..., prerouter=None,+ lora="")` 生效;未知字段抛 `TypeError`。+- **适配器契约**(test_registry.py:89–92):断言 `MODEL_REGISTRY` 每个条目都暴露+ `Config` / `build_model` / `build_engine`。+- **路径解析**(test_moe_spec.py):`keys` 模板、`block_of` 数字段下标、`layer_of`+ 从 `block_path` 推出的默认、`bundle_projs` 随布局切换。+- **数学 parity**:路由数学必须与 vendored 模型 bit 级一致(`moe/routing.py` 用 parity+ 测试钉死),若你的模型引入新路由种类需补对照。++## 完整最小骨架(对照 edge0-35b)++```python+# src/edge0/models/my_model/__init__.py+import sys+from edge0.models.base import ModelConfig+from edge0.moe.spec import MoESpec, QuantSpec, RouterKind, WeightLayout+from edge0.registry import register_model++class MyConfig(ModelConfig):+ @classmethod+ def _defaults(cls, model_dir):+ return cls(+ name="my-model",+ model_dir=model_dir,+ moe_spec=MoESpec(+ num_experts=..., top_k=..., intermediate_size=...,+ router=RouterKind.SOFTMAX_TOPK,+ quant=QuantSpec(bits=4, group_size=64, mode="affine"),+ layout=WeightLayout.SEPARATE,+ key_template="model.layers.{layer}.mlp.experts",+ block_path="model.layers.{layer}.mlp",+ layer_path="model.layers.{layer}",+ ),+ options=LayerOptions.staged_k4(), # 你的档位用 staged_k4 / staged_k8 等预设+ prerouter=...,+ )++def build_model(model_dir=None, **overrides):+ cfg = MyConfig.from_pretrained(model_dir, **overrides)+ return ... # load_installed 或你的装配路径++def build_engine(model_dir=None, **overrides):+ cfg = MyConfig.from_pretrained(model_dir, **overrides)+ return ... # 你的引擎++register_model("my-model", sys.modules[__name__])+Config = MyConfig+```++## 参考++- `src/edge0/registry.py` —— 注册表 + Auto 三件套+- `src/edge0/models/base.py` —— `ModelConfig` / `from_pretrained` / artifact 解析+- `src/edge0/models/edge0_35b/__init__.py`、`edge0_10b/__init__.py` —— 两个活例子+- `tests/test_registry.py`、`tests/test_moe_spec.py` —— 行为示例diff --git a/docs/architecture.md b/docs/architecture.mdnew file mode 100644index 0000000..1eb557f--- /dev/null+++ b/docs/architecture.md@@ -0,0 +1,136 @@+# 架构总览++**edge0** 是一个面向 Apple Silicon 的开源流式 MoE 推理框架,把生产部署中验证过的+「SSD 专家 offload + 并行 LoRA + prerouter 路由预判」方案抽象成可扩展的通用框架。+开箱支持两个模型:`edge0-35b`(Qwen3.5-MoE,K=4)与 `edge0-10b`(Ling 3.0+hybrid,K=8)。++## 核心设计目标++1. **像 transformers 一样使用**:`AutoConfig` / `AutoModel` / `AutoEngine` 按模型+ 名(或 checkpoint 的 `model_type`)自动解析适配器;+2. **后端隔离**:全部 MLX 代码收在 `edge0/backends/mlx/`,框架代码只通过+ `edge0.backends` 暴露的 `core` / `nn` / `io` / `quant` 命名空间接触后端,+ 为未来 CUDA 后端预留平级插槽(`backends/cuda/`,由 `EDGE0_BACKEND` 环境变量+ 选择);+3. **适配器统一为 safetensors**:LoRA 与 prerouter 权重均为带元数据的+ `.safetensors`,旧 npz 训练导出只经一次性迁移脚本转换后即弃用;+4. **术语统一**:预路由头一律称 **prerouter**,代码与文档零旧术语残留。++## 分层结构++```+src/edge0/+├── backends/ # 后端抽象(唯一允许接触 MLX 的边界)+│ ├── base.py # TensorStore 协议 + open_tensor_store()+│ ├── __init__.py # 按 EDGE0_BACKEND 装配 core/nn/io/quant+│ └── mlx/ # MLX 参考实现(_impl/ 为 vendored 模型)+├── config.py # GenerationConfig(采样参数)+├── sampling.py # 采样器(温度/top-k/top-p/重复惩罚,经 backends.core)+├── registry.py # 模型注册表 + AutoConfig/AutoModel/AutoEngine+├── attention/spec.py # 注意力规格(MHA/MLA 差异的抽象层)+├── moe/ # MoE 抽象+│ ├── spec.py # MoESpec/QuantSpec/RouterKind/WeightLayout+│ └── routing.py # 路由数学(与 vendored 模型 bit 级一致)+├── streaming/ # SSD 流式专家层+│ ├── options.py # LayerOptions(typed 替换部署期 env 旋钮)+│ ├── layer.py # StreamingSwitchGLU(每层状态机,全部执行路径)+│ ├── mmap.py # SafetensorsMmap(byte-range mmap)+│ └── cache.py # SharedExpertCache(跨层 LRU)+├── prerouter/ # 路由预判+│ ├── spec.py # PrerouterSpec(纯配置)+│ ├── install.py # 头权重安装+│ ├── heads.py # 头数学(fc1 -> erf gelu -> fc2 + linear_init)+│ ├── stager.py # 跨 token 预测保存/提交(每家族子类)+│ └── state.py # 跨 token 状态+├── engine/ # 推理编排(prefill/decode 循环,按模型子类化)+│ ├── base.py # 共享循环(经 backends.core)+│ ├── qwen.py # Qwen35Engine(K=4 档)+│ └── ling.py # Ling10BEngine(K=8 档)+├── adapters/lora.py # 并行 LoRA 安装(backends.nn)+├── models/ # 模型适配层(transformers 风格)+│ ├── base.py # ModelConfig 基类 + artifact 路径解析+│ ├── edge0_35b/ # edge0-35b 档(注册名 "edge0-35b")+│ └── edge0_10b/ # edge0-10b 档(注册名 "edge0-10b")+└── server/ # OpenAI 兼容 HTTP 服务(纯 stdlib)+ ├── app.py # ThreadingHTTPServer + 路由+ └── chat.py # /v1/chat/completions 会话状态+```++### 依赖方向++```+server / cli+ └─ engine ──> streaming / prerouter / adapters ──> moe / attention spec+ └──────────────> backends.{core,nn,io,quant}+ └─> backends/mlx/(MLX 唯一入口)+```++纯配置层(`config.py`、`moe/spec.py`、`prerouter/spec.py`、`streaming/options.py`、+`registry.py`、`models/*`、`server/`)不 import 任何后端包;运行时逻辑一律通过+`edge0.backends` 命名空间。CI 用 grep 强制:MLX 包不得在 `backends/mlx/` 之外被+import。++## 关键数据流++### Prefill(整层装载)++对前 `prefill_full_layers` 个层:`load_full_layer()` 把整层 9 个张量(gate/up/down+× weight/scales/biases)按 bundle 布局装入,`_gather_sort` + 排序 gather + 反排序+一次前向算完,随后 `clear_full_layer()` 释放,页缓存承担热数据(E3b 策略)。+其余层走 hot-stack(LRU 常驻 top-N 专家)或 on-demand exact 路径。++### Decode(staged 双缓冲)++1. prerouter 头在 token t-1 预测 token t 的专家集;+2. `stager` 把预测写入双缓冲槽;step 边界同步/异步填充槽位;+3. `StreamingSwitchGLU` 用槽位表 + 增量栈(`incr_stack`)把 9 个+ `mx.stack` 图节点换成本地 `put_along_axis` 行写入;+4. 专家权重经 `quant.gather_qmm` 按槽位 gather,路由索引全程不离开 GPU。++### 采样与生成++`engine/base.py` 的共享循环驱动 prefill → decode,逐 token 调+`sampling.sample()`(温度/top-k/top-p/重复惩罚,向量化历史惩罚,单次+host-sync categorical 抽样)。++## 注册表与模型接入++适配器模块暴露三件套(契约):++- `Config` —— `ModelConfig` 子类(`from_pretrained` 合并 override);+- `build_model(model_dir, **overrides)` —— 装配好的模型骨架;+- `build_engine(model_dir, **overrides)` —— 可生成引擎。++`register_model(name, module)` 注册;`TYPE_ALIASES` 把 checkpoint+`config.json` 的 `model_type`(如 `qwen3_5_moe`、`bailing_hybrid`)映射到注册名,+因此 `AutoEngine.from_pretrained(model_dir=...)` 可以直接从目录解析。+详见 [adding-a-model.md](adding-a-model.md)。++## 后端隔离的落法++`edge0.backends` 是框架唯一允许出现的后端 import 面:++- `core` —— 数组与算子命名空间(matmul/softmax/topk/take/eval/compile/random 等);+- `nn` —— 模块工厂(Module/Linear/RMSNorm/silu/gelu);+- `io` —— 模型/分词器/张量库加载;+- `quant` —— 量化 gather 内核(`gather_qmm`)。++MLX 后端是参考实现;未来 CUDA 后端实现同一表面即可复用全部框架代码。+注意:**每个模型的引擎胶水**(`engine/qwen.py`、`engine/ling.py`)因绑定 vendored+后端模型而引用 `edge0.backends.mlx._impl`,这是有意的例外——模型本身就是后端+资产,编排循环(`engine/base.py`)才是跨后端共享的。++## 测试策略++- `tests/test_streaming_math.py` —— 数学契约:exact/staged/hot/full 各路径与+ dequant 参考逐元素对比(相对 L2 < 1%),防 gate/up 交换、行错位类回归;+- `tests/test_moe_spec.py`、`tests/test_registry.py`、`tests/test_sampling.py` ——+ 纯逻辑层单测,无需真实 checkpoint;+- 验收指标:`target_tok_s` 与 `peak_active_mem_mb` 记录在生产机器上实测的+ 吞吐与峰值活跃内存,作为回归标尺。++## License++Apache-2.0,含 vendored 第三方模型代码(mlx-lm 的 Qwen3.5-MoE、生产部署的+ling backbone),详见根目录 [NOTICE](../NOTICE)。diff --git a/docs/attention.md b/docs/attention.mdnew file mode 100644index 0000000..84fa02e--- /dev/null+++ b/docs/attention.md@@ -0,0 +1,126 @@+# 注意力规格(AttentionSpec)++`edge0` 不重新实现注意力内核:具体的序列混合计算由 **vendored 基础模型**携带+(`edge0.backends.mlx._impl` 中的 GQA / GatedDeltaNet 对应 edge0-35b,+MLA / DeltaNet 对应 edge0-10b)。注意力内核属于**后端模型资产**,不属于框架。++那为什么还要一个 `AttentionSpec`?因为框架需要在**不依赖具体实现**的情况下+**审视(introspect)**一个模型:缓存尺寸、层角色、文档、未来的内核适配。+`AttentionSpec` 就是这层「注意力分类学」的载体。本文档说明它是什么、字段语义、+引擎如何使用,以及为什么单独抽象(MLA/MHA 的差异)。++源码:`src/edge0/attention/spec.py`。++## 设计动机++模块头部的 docstring 明确了边界:++```+edge0 does not re-implement attention kernels: the vendored base model+implementations (edge0.backends.mlx._impl) carry the kernels (GQA /+GatedDeltaNet for edge0-35b, MLA / DeltaNet for edge0-10b). This module+describes attention so the framework can introspect a model — cache+sizing, layer roles, documentation, and future kernels — without knowing+the concrete implementation.+```++新增一种注意力类型 = 一个新的 `AttentionKind` 成员 + 一份 `AttentionSpec` 描述 ++一个内核实现(在 vendored 基础模型或某个后端的 `_impl` 模块里)。++## 注意力种类:`AttentionKind`++`AttentionKind` 是 `str` 枚举,标识 edge0 支持的注意力 / 序列混合家族。++| 成员 | 值 | 含义 |+| --- | --- | --- |+| `GQA` | `"gqa"` | 分组查询注意力,softmax + KV cache |+| `MLA` | `"mla"` | 多头潜在注意力(压缩 KV) |+| `DELTANET` | `"deltanet"` | DeltaNet 线性注意力(门控 delta 规则,无 KV cache) |+| `GATED_DELTANET` | `"gated_deltanet"` | GatedDeltaNet(带衰减的门控 delta / 线性注意力) |+| `DENSE_MLP` | `"dense_mlp"` | 无序列混合(纯 MLP 层) |++`DENSE_MLP` 的存在说明:一个模型的某些层可能根本没有注意力块(例如 edge0-10b 的+layer 0 是 dense 层),分类学要能显式表达「这一层不做序列混合」。++## 规格:`AttentionSpec`++`AttentionSpec` 是一个 `frozen=True` 的 dataclass,描述**一个层**的序列混合块。++| 字段 | 类型 | 默认值 | 语义 |+| --- | --- | --- | --- |+| `kind` | `AttentionKind` | (必填) | 注意力家族 |+| `layer_indices` | `tuple[int, ...]` | (必填) | 该规格覆盖的 0 基层号 |+| `num_heads` | `int \| None` | `None` | 头数(仅 GQA/MLA;否则为 `None`) |+| `num_kv_heads` | `int \| None` | `None` | KV 头数(仅 GQA/MLA) |+| `head_dim` | `int \| None` | `None` | 每头维度(仅 GQA/MLA) |+| `cache` | `bool` | `True` | 该块是否维护 KV 式缓存(线性注意力层为 `False`) |+| `notes` | `str` | `""` | 自由说明(例如混合调度计划) |++要点:++- `cache` 字段区分了「softmax 注意力(要 KV cache)」与「线性注意力(无 KV cache)」,+ 这是缓存尺寸计算的关键分叉。+- `layer_indices` 是一个元组而非单层,因为**一个规格可以覆盖多个层**(例如一个+ GatedDeltaNet 规格可以描述连续的若干层),也便于表达混合调度。+- `__repr__` 被设计为**紧凑单行**,便于打日志 / 文档,例如:+ `AttentionSpec(gated_deltanet, layers=0..19, cache=False)`。++### 汇总:`summarize()`++```python+def summarize(specs: list[AttentionSpec]) -> str+```++把一份规格列表压缩成一行人类可读摘要,例如用于 CLI 启动横幅:++```+gated_deltanetx20, dense_mlpx1+```++即 `{kind.value}x{len(layer_indices)}` 的逗号拼接。文档标注它用于 CLI banner。++## 引擎如何使用++当前源码里,`AttentionSpec` 及其辅助函数**尚未被任何引擎 / 模型代码引用**——它是一份+「先定义、后接线」的分类学模块(`grep` 全仓仅命中 `attention/spec.py` 自身)。+这一点需要诚实标注:`AttentionKind` / `AttentionSpec` / `summarize` 目前的消费方只有+本模块,属于面向未来的框架设施,其设计意图(缓存尺寸、层角色、文档、未来内核)由+docstring 与字段定义承载。++即便如此,抽象方向已经明确:++- **缓存尺寸**:`cache` 字段让框架无需理解内核细节即可判断该层要不要分配 KV cache、+ 以及(对 GQA/MLA)按 `num_heads` / `num_kv_heads` / `head_dim` 计算尺寸。+- **层角色**:`kind` 把「softmax 注意力 / 线性注意力 / 无混合」区分开,供调度、文档与+ 性能剖析参考。+- **未来内核**:新增注意力类型无需改动框架其余部分,只要新加枚举成员 + 规格 + 内核。++这与后端抽象(`edge0.backends`)是一致的思想:框架代码只依赖抽象的规格 / 命名空间,+具体数学由后端或 vendored 模型实现。++## 为什么单独抽象(MLA / MHA 差异)++与其说「引擎在跑 attention」,不如说引擎在编排「一层序列混合」。把注意力显式抽象出来,+是为了容纳差异巨大的实现而不改框架:++| 维度 | MHA / GQA(softmax) | MLA / 线性注意力 |+| --- | --- | --- |+| KV 表示 | 完整缓存,随层增长 | 压缩潜在 KV |+| 缓存需求 | 需要 KV cache(`cache=True`) | 无 / 极简缓存(`cache=False`) |+| 头几何 | `num_heads` / `num_kv_heads` / `head_dim` 有意义 | 通常不适用(`None`) |+| 混合层级 | 几乎每层 | 可能只有部分层(含 dense 层) |++`AttentionSpec` 用统一的字段集表达这两类差异:`kind` 表达家族,`cache` 表达缓存有无,+头几何字段仅在 GQA/MLA 下填充,`layer_indices` 表达混合层级与调度。这样框架对+edge0-35b 的 GatedDeltaNet 与 edge0-10b 的 MLA 可以走同一套审视逻辑,而不必各自+硬编码。++## 新增一种注意力的步骤++按模块 docstring,新增 `Xxx` 注意力:++1. 在 `AttentionKind` 加一个成员;+2. 写一份 `AttentionSpec` 描述;+3. 提供内核实现(vendored 基础模型或某后端 `_impl` 模块)。++以上即 `src/edge0/attention/spec.py` 的全部内容与意图。diff --git a/docs/models/edge0-10b.md b/docs/models/edge0-10b.mdnew file mode 100644index 0000000..ec37560--- /dev/null+++ b/docs/models/edge0-10b.md@@ -0,0 +1,163 @@+# edge0-10b++edge0 平台的高吞吐轻量档:基于 Ling 3.0 混合架构(MLA + MoE)的 10B 级稀疏混合专家模型。相比 35b 档牺牲一定的参数量,换来更低的峰值内存与更高的生成速率,适合对延迟与显存敏感的场景。++生产档案基于 v7 checkpoint 实测,默认由 `Ling10BConfig`(`src/edge0/models/edge0_10b/__init__.py`)固定。++## 模型档案++| 项目 | 值 |+| --- | --- |+| 参数量级 | 10B 级 |+| 层数 | 24(第 0 层为 dense) |+| 专家数 | 128(路由专家)+ 1 常驻共享专家 |+| top_k(K) | 8(原生路由宽度) |+| 路由方式 | `SIGMOID_GROUP`(sigmoid + group 限制 top-k:`n_group=8, topk_group=4, routed_scaling=2.5, norm_topk_prob=True`) |+| 专家量化 | 4-bit affine,group 64 |+| 权重布局 | `WeightLayout.SEPARATE`(gate/up/down 分离张量堆叠) |+| 专家权重路径 | `model.layers.N.mlp.experts` |+| 前置路由(prerouter) | 22 个头(显式 owners 1..22),start_layer 1,hidden 512,fp16 |+| 解码调用方式 | `patch_call=False`(头内建在 `BailingSparseMoE` 中,从 `prerouter_cache` logits 消费) |+| LoRA | `r=16, alpha=32.0` |+| 预填分块 | 2048 |+| 热窗 | 1 |+| 流式预取历史 | 开(`prefetch_history=True`) |+| 服务端口 | 8083 |+| 验收吞吐 | ≈ 33 tok/s |+| 验收峰值激活内存 | ≈ 1.4 GB |++> 说明:数值全部取自 `Ling10BConfig._defaults()` 与 `LayerOptions.staged_k8()`。头部数量来自显式 `owners=range(1, 23)`,共 22 个头;`feature_topk="executed"`。++## 分阶段解码(staged decode)++该档使用 `LayerOptions.staged_k8()` 预设:++- 固定槽分阶段解码(`staged=True`,`staged_n=8`,`staged_trigger=8`,`staged_sync=True`),逐层无 host 同步。+- `staged_replace=False`:路由由内建的混合 prerouter 提供,分阶段集合与路由集合一致,槽表映射零丢弃。+- 不使用热专家钉住(`hot_per_layer=0`),`prefill_hot=0`。+- 整层 E3b prefill(`full_layer_prefill=True`)。++## 使用方式++### CLI 起服务++```bash+edge0 serve --name edge0-10b --model-dir /path/to/checkpoint --host 127.0.0.1 --port 8083+```++可选参数:++- `--no-prerouter`:禁用前置路由(`prerouter=None`)。+- `--no-lora`:禁用 LoRA(`lora=""`)。+- `--flask`:改用 Flask 传输(需要安装 flask,支持 SSE 流式)。++单轮对话(终端)可改用 `chat`:++```bash+edge0 chat --name edge0-10b --model-dir /path/to/checkpoint --prompt "你好"+```++查看该档默认档案:++```bash+edge0 models+```++### Python API++```python+from edge0 import AutoEngine, AutoModel, AutoConfig++# 直接可生成的引擎+engine = AutoEngine.from_pretrained(+ "/path/to/checkpoint", name="edge0-10b",+)+ids = engine.generate([156895]) # 内部使用配置里的默认采样+text = engine._tok.decode(ids)+engine.close()++# 仅加载权重(流式专家 + prerouter + LoRA 已装配)+model = AutoModel.from_pretrained("/path/to/checkpoint", name="edge0-10b")++# 仅拿配置+cfg = AutoConfig.from_pretrained("/path/to/checkpoint", name="edge0-10b")+```++`AutoEngine` / `AutoModel` / `AutoConfig` 三者也可省略 `name`,从 checkpoint 的 `config.json` 的 `model_type` 或目录 basename 自动解析(见 `src/edge0/registry.py`)。++## HTTP API++`edge0 serve` 暴露一个 OpenAI 兼容的单模型端点。引擎一次独占一个请求,生成以 FIFO 队列串行执行。++| 方法 | 路径 | 说明 |+| --- | --- | --- |+| `GET` | `/healthz` | 健康检查 |+| `GET` | `/v1/models` | 列出已加载模型 |+| `POST` | `/v1/chat/completions` | 对话补全(支持 `stream`) |+| `POST` | `/v1/completions` | 不支持,返回 400 |++### 非流式对话++```bash+curl -s http://127.0.0.1:8083/v1/chat/completions \+ -H 'Content-Type: application/json' \+ -d '{+ "model": "edge0-10b",+ "messages": [{"role": "user", "content": "介绍一下 Ling"}],+ "temperature": 0.7,+ "max_tokens": 256+ }'+```++响应字段(非流式):++```json+{+ "id": "chatcmpl-...",+ "object": "chat.completion",+ "created": 1750000000,+ "model": "edge0-10b",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "..."},+ "finish_reason": "stop"+ }],+ "usage": {"prompt_tokens": 12, "completion_tokens": 30, "total_tokens": 42}+}+```++请求可选字段:`model`、`messages`(含 `role`/`content`,content 支持多段文本自动拼接)、`temperature`、`top_p`、`top_k`、`max_tokens`、`seed`、`stream`。++### 流式对话(需 Flask)++```bash+curl -N http://127.0.0.1:8083/v1/chat/completions \+ -H 'Content-Type: application/json' \+ -d '{+ "model": "edge0-10b",+ "messages": [{"role": "user", "content": "数到五"}],+ "stream": true+ }'+```++每个 token 输出一段 `data: {"object":"chat.completion.chunk", ...}` SSE 事件,结束以 `data: [DONE]` 收尾。++## 配置覆盖++`from_pretrained` 支持对任意公开字段做覆盖(未知字段会抛 `TypeError`)。++```python+from edge0 import AutoEngine++engine = AutoEngine.from_pretrained(+ "/path/to/checkpoint",+ name="edge0-10b",+ port=9083, # 覆盖默认端口 8083+ target_tok_s=35.0, # 覆盖验收吞吐目标+ prerouter=None, # 关闭前置路由+ lora="", # 关闭 LoRA+ prefill_chunk=1024, # 缩小预填分块+)+```++CLI 里对应的覆盖是 `--no-prerouter` / `--no-lora`(见 `src/edge0/cli.py` 的 `_engine_kwargs`)。引擎参数覆盖请直接走 `Ling10BConfig.from_pretrained(model_dir, **overrides)`。diff --git a/docs/models/edge0-35b.md b/docs/models/edge0-35b.mdnew file mode 100644index 0000000..c9b8a14--- /dev/null+++ b/docs/models/edge0-35b.md@@ -0,0 +1,163 @@+# edge0-35b++edge0 平台的第一档主力模型:基于 Qwen3.5-MoE(K=4 档)的 35B 级稀疏混合专家模型。通过流式专家加载(streaming experts)、训练前置路由(prerouter)与 LoRA 适配,把整份权重量化后驻留在磁盘、按需装载,在一块消费级设备上即可服务。++生产档案基于 round-7 checkpoint 实测,默认由 `Qwen35Config`(`src/edge0/models/edge0_35b/__init__.py`)固定。++## 模型档案++| 项目 | 值 |+| --- | --- |+| 参数量级 | 35B 级 |+| 层数 | 40 |+| 专家数 | 256(路由专家)+ 1 常驻共享专家 |+| top_k(K) | 4 |+| 路由方式 | `SOFTMAX_TOPK`(softmax → top-k → renormalize,`norm_topk_prob=True`) |+| 专家量化 | 4-bit affine,group 64 |+| 权重布局 | `WeightLayout.SEPARATE`(gate/up/down 分离张量堆叠) |+| 专家权重路径 | `language_model.model.layers.N.mlp.switch_mlp` |+| 前置路由(prerouter) | 33 个头(owners 6..38),start_layer 7,hidden 512,fp16 |+| 解码调用方式 | `patch_call=True`,跨 token 分阶段解码,K=4 |+| LoRA | `r=16, alpha=32.0` |+| 预填分块 | 2048 |+| 热窗 | 4 |+| 流式预取历史 | 开(`prefetch_history=True`) |+| 服务端口 | 8085 |+| 验收吞吐 | ≈ 13 tok/s |+| 验收峰值激活内存 | ≈ 3.3 GB |++> 说明:数值全部取自 `Qwen35Config._defaults()` 与 `LayerOptions.staged_k4()`。头部数量来自显式 `owners` 列表(6 到 38,共 33 个头);`feature_topk="executed"` 表示喂入头的 top-k 特征即解码时实际路由的集合。++## 分阶段解码(staged decode)++该档使用 `LayerOptions.staged_k4()` 预设:++- 固定槽分阶段解码(`staged=True`,`staged_n=4`,`staged_sync=True`),逐层无 host 同步。+- `staged_replace=False`:路由由训练好的 prerouter 头提供(MoE 块经由 prerouter logits 路由),因此分阶段集合与路由集合完全一致,槽表映射零丢弃。+- 常驻热专家钉住(`hot_per_layer=32`),预热钉每 4 次 forward 刷新。+- 整层 prefill(`full_layer_prefill=True`),前 12 层整层装载(E3b),prefill 热栈 32。++## 使用方式++### CLI 起服务++```bash+edge0 serve --name edge0-35b --model-dir /path/to/checkpoint --host 127.0.0.1 --port 8085+```++可选参数:++- `--no-prerouter`:禁用前置路由(`prerouter=None`)。+- `--no-lora`:禁用 LoRA(`lora=""`)。+- `--flask`:改用 Flask 传输(需要安装 flask,支持 SSE 流式)。++单轮对话(终端)可改用 `chat`:++```bash+edge0 chat --name edge0-35b --model-dir /path/to/checkpoint --prompt "你好"+```++查看该档默认档案:++```bash+edge0 models+```++### Python API++```python+from edge0 import AutoEngine, AutoModel, AutoConfig++# 直接可生成的引擎+engine = AutoEngine.from_pretrained(+ "/path/to/checkpoint", name="edge0-35b",+)+ids = engine.generate([248044]) # 内部使用配置里的默认采样+text = engine._tok.decode(ids)+engine.close()++# 仅加载权重(流式专家 + prerouter + LoRA 已装配)+model = AutoModel.from_pretrained("/path/to/checkpoint", name="edge0-35b")++# 仅拿配置+cfg = AutoConfig.from_pretrained("/path/to/checkpoint", name="edge0-35b")+```++`AutoEngine` / `AutoModel` / `AutoConfig` 三者也可省略 `name`,从 checkpoint 的 `config.json` 的 `model_type` 或目录 basename 自动解析(见 `src/edge0/registry.py`)。++## HTTP API++`edge0 serve` 暴露一个 OpenAI 兼容的单模型端点。引擎一次独占一个请求,生成以 FIFO 队列串行执行。++| 方法 | 路径 | 说明 |+| --- | --- | --- |+| `GET` | `/healthz` | 健康检查 |+| `GET` | `/v1/models` | 列出已加载模型 |+| `POST` | `/v1/chat/completions` | 对话补全(支持 `stream`) |+| `POST` | `/v1/completions` | 不支持,返回 400 |++### 非流式对话++```bash+curl -s http://127.0.0.1:8085/v1/chat/completions \+ -H 'Content-Type: application/json' \+ -d '{+ "model": "edge0-35b",+ "messages": [{"role": "user", "content": "介绍你自己"}],+ "temperature": 0.6,+ "max_tokens": 256+ }'+```++响应字段(非流式):++```json+{+ "id": "chatcmpl-...",+ "object": "chat.completion",+ "created": 1750000000,+ "model": "edge0-35b",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "..."},+ "finish_reason": "stop"+ }],+ "usage": {"prompt_tokens": 12, "completion_tokens": 30, "total_tokens": 42}+}+```++请求可选字段:`model`、`messages`(含 `role`/`content`,content 支持多段文本自动拼接)、`temperature`、`top_p`、`top_k`、`max_tokens`、`seed`、`stream`。++### 流式对话(需 Flask)++```bash+curl -N http://127.0.0.1:8085/v1/chat/completions \+ -H 'Content-Type: application/json' \+ -d '{+ "model": "edge0-35b",+ "messages": [{"role": "user", "content": "数到五"}],+ "stream": true+ }'+```++每个 token 输出一段 `data: {"object":"chat.completion.chunk", ...}` SSE 事件,结束以 `data: [DONE]` 收尾。++## 配置覆盖++`from_pretrained` 支持对任意公开字段做覆盖(未知字段会抛 `TypeError`)。++```python+from edge0 import AutoEngine++engine = AutoEngine.from_pretrained(+ "/path/to/checkpoint",+ name="edge0-35b",+ port=9090, # 覆盖默认端口 8085+ target_tok_s=14.0, # 覆盖验收吞吐目标+ prerouter=None, # 关闭前置路由+ lora="", # 关闭 LoRA+ prefill_chunk=1024, # 缩小预填分块+)+```++CLI 里对应的覆盖是 `--no-prerouter` / `--no-lora`(见 `src/edge0/cli.py` 的 `_engine_kwargs`)。引擎参数覆盖请直接走 `Qwen35Config.from_pretrained(model_dir, **overrides)`。diff --git a/docs/moe.md b/docs/moe.mdnew file mode 100644index 0000000..8c74172--- /dev/null+++ b/docs/moe.md@@ -0,0 +1,188 @@+# MoE 规格与路由(MoESpec / Routing)++MoE 抽象位于 `src/edge0/moe/`,两个文件:++- `spec.py` —— 路由种类(`RouterKind`)、量化规格(`QuantSpec`)、权重布局+ (`WeightLayout`),以及驱动**驻留数学与流式层**的 `MoESpec`;+- `routing.py` —— 两种路由数学,与 vendored 模型 **bit 级一致**(从它们逐字抽取)。++设计核心:**每个 MoE 块由一份 `MoESpec` 描述**,流式子系统消费这份规格(布局 ++key 模板),因此**一个通用的 `StreamingSwitchGLU` 就能服务所有模型**。++## 路由种类:`RouterKind`++`RouterKind` 是 `str` 枚举,edge0 支持两类路由数学家族。++| 成员 | 值 | 语义 |+| --- | --- | --- |+| `SOFTMAX_TOPK` | `"softmax_topk"` | 精确 softmax → top-k → 重新归一化(Qwen3.5-MoE 风格,`norm_topk_prob=True`) |+| `SIGMOID_GROUP` | `"sigmoid_group"` | sigmoid 分数 + 分组限 top-k + routed 缩放(DeepSeek-V3 / Bailing 风格) |++两个家族对应两份模型:edge0-35b 用 `SOFTMAX_TOPK`,edge0-10b 用 `SIGMOID_GROUP`。+对应数学实现见「路由函数」一节。++## 量化规格:`QuantSpec`++`QuantSpec` 是 `frozen` dataclass,描述**逐张量的默认**权重量化。++| 字段 | 类型 | 默认值 | 语义 |+| --- | --- | --- | --- |+| `bits` | `int` | `4` | 量化位宽 |+| `group_size` | `int` | `64` | 量化组大小 |+| `mode` | `str` | `"affine"` | 量化模式(仿射) |++两个生产模型都使用 `bits=4, group_size=64, mode="affine"`。流式层的量化 gather+(`quant.gather_qmm`)消费这三个字段。++## 权重布局:`WeightLayout`++`WeightLayout` 枚举描述专家投影在 checkpoint 里的存储方式。++| 成员 | 值 | 语义 |+| --- | --- | --- |+| `SEPARATE` | `"separate"` | `gate_proj` / `up_proj` / `down_proj` 三份堆叠张量 |+| `FUSED_GATE_UP` | `"fused_gate_up"` | `gate_proj + up_proj` 融合成一份堆叠张量(out 轴行数加倍);`down_proj` 独立 |++当前两个模型都用 `SEPARATE`;`FUSED_GATE_UP` 为支持融合 checkpoint 的模型预留。++## 规格:`MoESpec`++`MoESpec` 是 `frozen` dataclass,描述一个模型所有 MoE 块需要的一切。++| 字段 | 类型 | 默认值 | 语义 |+| --- | --- | --- | --- |+| `num_experts` | `int` | (必填) | 每层被路由的专家数 |+| `top_k` | `int` | (必填) | 每 token 选择的专家数(路由宽度) |+| `intermediate_size` | `int` | (必填) | 专家 FFN 隐层尺寸(**仅文档**;数学从权重张量推导尺寸) |+| `router` | `RouterKind` | `SOFTMAX_TOPK` | 路由数学家族 |+| `norm_topk_prob` | `bool` | `True` | 是否对选择权重重新归一化 |+| `routed_scaling` | `float \| None` | `None` | 施加在选择权重上的缩放系数(sigmoid-group 路由);softmax-topk 为 `None` |+| `n_group` | `int \| None` | `None` | 分组路由的组数(仅 sigmoid-group) |+| `topk_group` | `int \| None` | `None` | 存活的分组数(仅 sigmoid-group) |+| `shared_experts` | `int` | `0` | 常驻共享专家数(**从不被流式**) |+| `quant` | `QuantSpec` | `QuantSpec()` | 权重量化 |+| `layout` | `WeightLayout` | `SEPARATE` | 权重布局 |+| `key_template` | `str` | `""` | safetensors key 前缀,含 `{layer}`;`{proj}` / `{part}` 占位由消费方追加 |+| `block_path` | `str` | `""` | 从加载的模型对象到 MoE 块的点分属性路径,含 `{layer}` |+| `layer_path` | `str` | `""` | 到 **decoder 层对象**(块的宿主)的点分路径,含 `{layer}`;prerouter stager 用它读每层缓存;默认由 `block_path` 去掉末尾属性段推出 |+| `expert_row_axis` | `int` | `0` | 堆叠 `[num_experts, ...]` 张量中,一个专家是连续切片的轴(受支持布局恒为 0,仅为文档保留) |++关键差异标注:++- `intermediate_size` 仅供文档——**数学不依赖它**,尺寸一律从权重张量推导。+- `shared_experts` 的语义是「常驻、永不流式」:共享专家留在基础模型里,只有路由专家被+ `StreamingSwitchGLU` 流式处理。++### Bundle 布局与 up-first 顺序(重点)++`bundle_projs` 属性返回**缓存专家 bundle 所含投影,按堆叠顺序**:++```python+if self.layout is WeightLayout.FUSED_GATE_UP:+ return ("gate_up_proj", "down_proj")+return ("up_proj", "gate_proj", "down_proj")+```++**unfused 顺序是 `("up_proj", "gate_proj", "down_proj")`——up 在前。**++这个顺序不是随意的,它**直接对齐数学参数的顺序**。GLU 激活的数学是:++```python+def _swiglu(up, gate):+ return nn.silu(gate) * up+```++即 `_swiglu(up, gate)`。因此权重栈把 `up` 放在 `gate` 之前,正好对应函数签名+`(up, gate)`——消费方只需按顺序喂参数,**不需要做任何重排**。`StreamingSwitchGLU`+的 bundle 构建与数学路径正是按 `self._bundle_projs` 的顺序展开的。++**fused 布局则把 gate 行叠在 up 行之上**(out 轴方向),所以++```python+x_gate, x_up = core.split(x_gu, 2, axis=-1)+```++一次 `split(x_gu, 2)` 恰好得到 `(gate, up)`。行序是「gate 在上、up 在下」。构建 fused+bundle 时,shard 里 gate/up 是两份独立张量,必须**按专家逐个交错**拼接(gate 切片在前、+up 切片在后),而非整体拼接,否则会行错位(见 `streaming/layer.py` 的 `_build`)。++`fuse_gu` 属性是 `layout is FUSED_GATE_UP` 的快捷判断,供流式层分支使用。++### 路径解析:`keys()` / `block_of()` / `layer_of()`++`MoESpec` 用三条路径字符串把「层号」翻译成真实对象 / 张量 key。++**`keys(layer, proj, part)`** —— 解析某层某投影某部分的一个 safetensors key:++```python+prefix = self.key_template.format(layer=layer)+return f"{prefix}.{proj}.{part}"+```++例(edge0-35b):`keys(3, "gate_proj", "weight")` 得到+`language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight`。++**`block_of(model, layer)`** —— 解析 MoE 块对象。沿 `block_path` 逐段 `getattr`,+遇到纯数字段则按下标取(多数家族里 layer 是普通 list)。++**`layer_of(model, layer)`** —— 解析 decoder 层对象(块的宿主)。若给了 `layer_path`+用它的模板;否则按约定回退:块位于 `<layer>.<mlp>.<block>`,层就是去掉末尾**两段**+属性后的路径(`split(".")[:-2]`)。prerouter stager 用 `layer_of` 读取每层缓存。++测试(`tests/test_moe_spec.py`)用假模型验证了这些路径解析:数字段走下标、`layer_path`+缺失时从 `block_path` 正确推出层路径、`bundle_projs` 随布局切换、`QuantSpec` 默认值、+两种 `RouterKind` 的值。++## 路由函数:`routing.py`++两个路由函数共享**驻留**与**流式**两条 MoE 路径。模块 docstring 强调:++> These functions must stay bit-identical to the vendored base+> implementations (they are extracted from them verbatim); parity tests pin+> this.++即它们是从 vendored 基础实现**逐字抽取**的,必须与之一致,由 parity 测试钉死。++### `select_from_logits(logits, top_k, norm=True)`++Softmax-topk 路由(Qwen3.5-MoE / `norm_topk_prob=True` 语义):++```python+gates = core.softmax(logits, axis=-1, precise=True)+inds = core.argpartition(gates, kth=-top_k, axis=-1)[..., -top_k:]+scores = core.take_along_axis(gates, inds, axis=-1)+if norm:+ scores = scores / scores.sum(axis=-1, keepdims=True)+return inds, scores+```++流程:**精确 softmax → top-k → (可选)重新归一化**。返回 `(inds [..., k], scores [..., k])`。++### `group_select_from_logits(logits, top_k, n_group, topk_group, routed_scaling, norm=True, expert_bias=None)`++Sigmoid + 分组限 top-k 路由(DeepSeek-V3 / Bailing 规则):++```python+scores = core.sigmoid(logits.astype(core.float32))+select = scores + expert_bias if expert_bias is not None else scores+```++- **选择分数**是 `sigmoid(logits) + expert_bias`;+- 按每组**前二**选择分数之和排序,保留 `topk_group` 个最优组,其余组分数置 `-inf`+ 排除(`k_drop = n_group - topk_group`;`topk_group == n_group` 时无组被丢);+- 在存活组内按选择分数取 top-k 专家;+- **权重**是所选专家的**原始 sigmoid 分数**(不含 bias),可选归一化+ (`w / (w.sum + 1e-20)`)后乘以 `routed_scaling`。++返回 `(inds [..., k], scores [..., k])`。注意「选谁」用选择分数(含 bias),「权重」用+raw sigmoid(不含 bias)——这是 DeepSeek 式路由的关键区分。++两份模型各取其一:edge0-35b 用 `select_from_logits`,edge0-10b 用+`group_select_from_logits`;prerouter stager 也复用这两个函数做跨 token 的教师路由。++## 参考++- `src/edge0/moe/spec.py` —— `MoESpec` / `QuantSpec` / `RouterKind` / `WeightLayout`+- `src/edge0/moe/routing.py` —— 两个路由函数+- `src/edge0/moe/__init__.py` —— 重新导出以上符号+- `tests/test_moe_spec.py` —— 路径解析与布局契约测试diff --git a/docs/prerouter.md b/docs/prerouter.mdnew file mode 100644index 0000000..8a39e0a--- /dev/null+++ b/docs/prerouter.md@@ -0,0 +1,71 @@+# prerouter:跨 token 路由预判++## 动机++MoE decode 每步都需要「本层输出 → 路由 → 下层的专家选择」,而路由依赖前一层+输出——SSD 流式下,等路由结果出来再加载专家意味着每步都要等权重装载。解法:+**用一层小网络提前一个 token 预测路由**,让专家装载与生成并行,staged 槽位+永远先于需求就绪。++## 双移位:prev-layer + prev-token++- prerouter 头 **OWNED by 层 N**:在 token t 用层 N 的 MoE 输入(attention 后+ norm 输出)预测**层 N+1** 的路由(层移位);+- 层 N+1 消费的是层 N 的头在 **token t-1** 的预测(token 移位)。++所以一个 decode step 的 staged 专家集就是 prerouter 的预测本身——**按构造零+drop**(`staged_replace` 语义)。++## 头结构(`prerouter/heads.py`)++特征向量:`concat[hidden, 本 token top-k one-hot, 上 token top-k one-hot]`+(`feature_topk` 决定 one-hot 来源:"executed" = 块实际路由的 top-k,qwen 训练+语义;"teacher" = 原 gate 重算的 top-k,ling v7 训练语义)。++```+head: fc1 -> exact(erf) gelu -> fc2 + linear_init (与训练导出 bit 级一致)+```++## 安装与接线(`prerouter/install.py`)++`install_prerouter(model, spec, store)`:++- 从 safetensors 读每个 owner 的头权重(键形如 `layers.<N>.fc1.weight`);+- 替换/注入模型中的 prerouter 模块;+- **patch_call**:qwen 的 vendored MoE 块不是 prerouter-aware,安装类级+ `__call__` 补丁让 decode 路由走 `pred_inds`;ling 的 `bailing_hybrid` 内置了+ 消费钩子,`patch_call=False`。++## 跨 token 分派(`prerouter/stager.py`)++`CrossTokenStager`(qwen)/ `LingPrerouterStager`(ling)在 step 边界把预测+**提交**给下一层:先保存本轮 logits,step 后按 `start_layer`/`owners` 把每个+owner 的预测写入对应 consumer 的 staged 槽位。家族差异通过三个 hook 表达:+`_features`(特征组装)、`_select`(logits → 专家选择)、`_store`(提交)。++## 与 streaming 的配合++- `LayerOptions.staged_replace=True`:路由集 == staged 集(prerouter 模式,+ 无 drop);+- 生产 profile(`staged_k4`/`staged_k8`)使用**显式索引**路由:prerouter 预测+ 直接作为 staged 填充来源,`staged_replace=False` 但槽表精确映射,同样零 drop;+- `pin_bonus`:prerouter 预测的专家在 hot 驻留选择时额外加分,让「即将用到」+ 的专家优先常驻。++## 模型档++| | edge0-35b | edge0-10b |+|---|---|---|+| 路由家族 | SOFTMAX_TOPK(precise softmax → top-k → 归一化) | SIGMOID_GROUP(sigmoid + 组限制 top-k,n_group 8 / topk_group 4,routed_scaling 2.5) |+| start_layer | 7(首个消费层) | 1 |+| heads(owners) | 33 个(6..38) | 22 个(1..22) |+| hidden | 512(fp16) | 512(fp16) |+| feature_topk | executed | executed |+| patch_call | True(qwen3_next 需补丁) | False(模型内置钩子) |+| 权重文件 | `artifacts/prerouter_edge0_35b_k4.safetensors` | `artifacts/prerouter_edge0_10b.safetensors` |++## 测试与回归++`tests/test_streaming_math.py` 的 staged 路径在「无 prerouter」下用显式索引+模拟槽位填充,保证 staged 数学与 exact 等价;prerouter 头本身的数值由+`heads.py` 的 bit 级移植保证(源自训练实现),引擎 e2e 冒烟验证真实生成链路。diff --git a/docs/streaming.md b/docs/streaming.mdnew file mode 100644index 0000000..7779b0a--- /dev/null+++ b/docs/streaming.md@@ -0,0 +1,109 @@+# SSD 流式专家层(streaming)++## 问题++Qwen3.5-MoE 有 40 层 × 256 专家,每层 MoE 权重约 310MB(4-bit 量化后)——全量+常驻远超 Apple Silicon 统一内存预算。edge0 的方案:**权重留在 SSD,按需 mmap+加载进 GPU,用 LRU + 预取 + 固定槽把「每步只动几十 MB」变成「每步几乎不动」**。++核心部件(`edge0/streaming/`):++| 模块 | 职责 |+|---|---|+| `mmap.py` | `SafetensorsMmap`:单文件 byte-range mmap,按张量名惰性读原始字节 |+| `cache.py` | `SharedExpertCache`:跨层共享的 LRU,容量 `cache_slots` |+| `layer.py` | `StreamingSwitchGLU`:每层一个实例,持有全部执行路径与状态 |+| `options.py` | `LayerOptions`:typed 配置(部署期 env 旋钮的替代品) |++## 数学契约(必须与 vendored 模型 bit 级一致)++MoE 块数学为 `down(silu(gate(x)) * up(x))`,即+`_swiglu(up, gate) = nn.silu(gate) * up`。**bundle 顺序直接对应数学参数顺序**,+权重栈不需要任何重排:++- **separate(未融合)**:bundle 顺序为+ `("up_proj", "gate_proj", "down_proj")` —— up 在前!堆栈后的 wargs 顺序与+ `_swiglu(up, gate)` 签名逐位对应:`(w_u, s_u, b_u, w_g, s_g, b_g, w_d, s_d, b_d)`。+- **fused gate+up**:bundle 顺序为 `("gate_up_proj", "down_proj")`;gate_up 行内+ 布局为 **gate 行在上、up 行在下**(沿输出特征轴拼接),`mx.split(x_gu, 2, -1)`+ 得到 `(gate, up)`。++任何一处重排(如历史上把 bundle 写成 gate-first)都会在+`tests/test_streaming_math.py` 的参考对比中暴露(相对 L2 从 ≈0.2% 跳到 ≈100%)。++### 量化布局++- `switch_mlp.<proj>.weight`:打包 u32 `[E, out, in/8]`(4-bit affine,组 64);+- `scales` / `biases`:bf16 位型,按组 `[E, out, in/64]`;+- 内核:`backends.quant.gather_qmm`(MLX 量化 gather matmul),dequant 后 bf16+ 内部精度(相对 L2 ≈0.24%,测试容差按此标定)。++## 执行路径(由快到全)++### 1. exact(on-demand bundle)++路由索引去重 → `_get_bundles(unique)` 逐专家构建(从 LRU 或 mmap 读)→ 堆栈 →+`gather_qmm`。最慢但最通用,是其他路径的正确性基准。++### 2. staged(固定槽双缓冲,decode 主力)++- `staged_n` 个固定槽位 + 溢出零槽;+- 路由索引 → `mx.take` 槽位表,**索引不离开 GPU**(每层每步零 host 同步);+- `staged_sync`:step 边界同步填充;`asm_cache`:按专家集缓存槽表与堆栈图节点,+ 重复集不重建;`incr_stack`:把 9 个 `mx.stack` 节点换成增量栈的+ `put_along_axis` 行写入(staged 期间从 LRU 去重,`incr_writeback` 归还);+- 缺专家映射到溢出零槽、贡献丢弃;槽未就绪时回退 exact 路径。++### 3. hot(LRU 常驻 top-N)++- 每层按衰减计数 `_hot_counts` 选 top-N(`hot_per_layer`),LRU 命中的专家以+ 常驻堆栈形式存在(`load_hot_layer` / `materialize_hot`,滑动窗口+ `hot_window` 控制同时驻留的层数);+- 命中走堆栈 gather,未命中走 exact 并 scatter-add。++### 4. full-layer(E3b 整层装载,prefill 主力)++- `load_full_layer()`:整层 9 个张量直接装入(checkpoint 本来就是逐层堆叠的+ 单张量,mmap 位型转换 ≈9ms/层,CPU 装载隐藏在上一层的 GPU 执行下);+- `_gather_sort` 把 batch 并入 token 维 → 排序 gather → `_scatter_unsort`+ 反排序并恢复 batch 维;+- `prefill_full_layers` 只对前导层整层装载(qwen 为 12 层),其余层走 hot/exact;+- 用后 `clear_full_layer()` 释放 GPU 副本,页缓存承担热数据。++## 排序与编译++`use_compile` 时,staged/exact 路径包进 `mx.compile`:++- 集合规模 ≥64 时用排序变体 `_moe_math_sorted`(`_gather_sort` 预处理 ++ 排序 gather + `_scatter_unsort` 还原);+- 否则用直接 gather 变体 `_moe_math`;+- 两者数学等价,测试对每条路径都做了 exact 对比。++## 关键选项(`LayerOptions`)++| 字段 | 含义 | 默认 |+|---|---|---|+| `staged` / `staged_n` / `staged_trigger` | 固定槽 decode 开关/槽数/触发 top-k | False / 8 / 8 |+| `staged_replace` | staged 集**取代**路由(配合 prerouter,零 drop) | False |+| `staged_sync` / `asm_cache` / `incr_stack` / `incr_writeback` | 填充同步 / 图节点缓存 / 增量栈 / 归还 LRU | True / True / False / False |+| `hot_per_layer` / `hot_update_interval` / `hot_decay` / `pin_bonus` | 热专家驻留数与刷新 | 0 / 4 / 0.75 / 2.0 |+| `cache_slots` / `prefetch_cap` | LRU 容量 / 预取缓冲 | 64 / 48 |+| `load_threads` / `prefetch_threads` | 构建 / 预取线程数 | 8 / 4 |+| `full_layer_prefill` / `prefill_full_layers` | 整层装载 prefill / 前导层数 | False / 0 |+| `prefill_hot` | prefill 期 hot 栈规模 | 0 |+| `use_compile` / `top_k` | 编译包装 / 路由 top-k 覆盖 | True / None |++预设:`staged_k4()`(edge0-35b:staged 4 槽 + hot 32 + 整层 prefill 12 层)、+`staged_k8()`(edge0-10b:staged 8 槽,无 hot 驻留)。两档共用+`cache_slots=64`。++## 为什么「整层」加载也快++checkpoint 的 MoE 权重本来就是逐层堆叠的单张量(`[256, 512, 256]` 这类),+整层装载是 9 次直接读 + 位型 view,不是 256×9 次逐专家构建。CPU 装载与 GPU+执行通过 `before_layer_cb` + `async_eval_per_layer` 流水线重叠。++> 注意 fused gate_up 的整层装载必须**逐专家交错** gate/up 行(与 `_build`+> 的逐专家拼接一致);把两个 `[E, ...]` 张量整体拼接再 reshape 会产生行错位+> (早期版本的真实 bug,已由 `test_full_layer_prefill_matches_exact[fused]`+> 钉死)。diff --git a/examples/demo.py b/examples/demo.pynew file mode 100644index 0000000..5fc3b0a--- /dev/null+++ b/examples/demo.py@@ -0,0 +1,50 @@+"""Minimal edge0 API walkthrough — the same path ``edge0 demo`` runs.++Install:+ python3.12 -m venv .venv && .venv/bin/pip install -e '.[dev]'++Run (checkpoint path or registered tier name):+ .venv/bin/python examples/demo.py --model-dir /path/to/qwen35/model+ .venv/bin/python examples/demo.py --model edge0-10b++Three lines do everything: build the engine (prerouter + LoRA + SSD+offload installed automatically from the tier config), template the+prompt, generate.+"""++from __future__ import annotations++import argparse+import sys++from edge0 import AutoEngine+from edge0.server.chat import ChatMessage, ChatRequest, ChatSession+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__)+ ap.add_argument("--model", default=None,+ help="registered tier name, e.g. edge0-35b")+ ap.add_argument("--model-dir", default=None,+ help="checkpoint directory (tier auto-detected)")+ ap.add_argument("--prompt",+ default="Hello! Write one short sentence about Zhuhai.")+ ap.add_argument("--max-new", type=int, default=24)+ args = ap.parse_args()++ engine = AutoEngine.from_pretrained(args.model_dir, name=args.model)+ req = ChatRequest(+ model=engine.name,+ messages=[ChatMessage(role="user", content=args.prompt)],+ max_tokens=args.max_new,+ )+ tokens, meta = ChatSession(engine, req).run()+ print(f"user : {args.prompt}")+ print(f"edge0: {engine._tok.decode(tokens)}")+ print(f"# {len(tokens)} tokens in {meta['wall_s']}s", file=sys.stderr)+ engine.close()+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/pyproject.toml b/pyproject.tomlnew file mode 100644index 0000000..3fe8d0b--- /dev/null+++ b/pyproject.toml@@ -0,0 +1,53 @@+[build-system]+requires = ["setuptools>=68"]+build-backend = "setuptools.build_meta"++[project]+name = "edge0"+version = "0.1.0"+description = "edge0: streaming MoE inference for Apple Silicon — SSD expert offload, parallel LoRA, prerouter routing prediction. Backend-isolated (MLX now, CUDA-ready)."+readme = "README.md"+requires-python = ">=3.10"+license = { text = "Apache-2.0" }+authors = [{ name = "edge0 contributors" }]+keywords = ["mlx", "moe", "inference", "lora", "ssd-offload", "streaming"]+classifiers = [+ "Development Status :: 3 - Alpha",+ "Intended Audience :: Developers",+ "License :: OSI Approved :: Apache Software License",+ "Programming Language :: Python :: 3",+ "Programming Language :: Python :: 3.10",+ "Programming Language :: Python :: 3.11",+ "Programming Language :: Python :: 3.12",+ "Topic :: Scientific/Engineering :: Artificial Intelligence",+]+dependencies = [+ # NOTE: mlx-lm is pinned to 0.31.0 on purpose. Newer releases crash in+ # decode with a `tolist()` on lazily-evaluated arrays. Do not upgrade+ # without re-running the full test suite.+ "mlx==0.30.4",+ "mlx-metal==0.30.4; platform_system == 'Darwin'",+ "mlx-lm==0.31.0",+ "numpy>=1.24",+ "safetensors>=0.4",+ "tokenizers>=0.15",+ "psutil>=5.9",+]++[project.optional-dependencies]+dev = [+ "pytest>=8.0",+]++[project.scripts]+edge0 = "edge0.cli:main"++[tool.setuptools.packages.find]+where = ["src"]++[tool.pytest.ini_options]+testpaths = ["tests"]+markers = [+ "slow: real-weight e2e tests (skip with -m 'not slow')",+]+addopts = "-m 'not slow'"diff --git a/scripts/alignment_results.json b/scripts/alignment_results.jsonnew file mode 100644index 0000000..19c58b0--- /dev/null+++ b/scripts/alignment_results.json@@ -0,0 +1,182 @@+{+ "demo_qwen": {+ "prompt_ids": [+ 248045,+ 846,+ 198,+ 9419,+ 0,+ 9357,+ 799,+ 2716,+ 11316,+ 883,+ 32246,+ 12180,+ 2075,+ 13,+ 248046,+ 198,+ 248045,+ 74455,+ 198,+ 248068,+ 198+ ],+ "out": [+ 90700,+ 8340,+ 25,+ 271,+ 16,+ 13,+ 220,+ 2972,+ 2014,+ 53983,+ 279,+ 5952,+ 64700,+ 198+ ],+ "text": "Thinking Process:\n\n1. **Analyze the Request:**\n"+ },+ "edge0_qwen": {+ "prompt_ids": [+ 248045,+ 846,+ 198,+ 9419,+ 0,+ 9357,+ 799,+ 2716,+ 11316,+ 883,+ 32246,+ 12180,+ 2075,+ 13,+ 248046,+ 198,+ 248045,+ 74455,+ 198,+ 248068,+ 198+ ],+ "out": [+ 90700,+ 8340,+ 25,+ 271,+ 16,+ 13,+ 220,+ 2972,+ 2014,+ 53983,+ 279,+ 5952,+ 64700,+ 198+ ],+ "text": "Thinking Process:\n\n1. **Analyze the Request:**\n"+ },+ "demo_ling": {+ "prompt_ids": [+ 157151,+ 90827,+ 157152,+ 14136,+ 5381,+ 6350,+ 928,+ 156895,+ 157151,+ 39,+ 116171,+ 157152,+ 34355,+ 7011,+ 26227,+ 5286,+ 40904,+ 311,+ 156895,+ 157151,+ 8469,+ 7342,+ 5468,+ 157152,+ 198,+ 156903,+ 156904+ ],+ "out": [+ 40904,+ 364,+ 6939,+ 1782,+ 24330,+ 77010,+ 1288,+ 1462,+ 9376,+ 290,+ 39738,+ 4334,+ 2589,+ 22703+ ],+ "text": "珠海是位于中国广东省珠江口西岸的沿海城市,以美丽的"+ },+ "edge0_ling": {+ "prompt_ids": [+ 157151,+ 90827,+ 157152,+ 14136,+ 5381,+ 6350,+ 928,+ 156895,+ 157151,+ 39,+ 116171,+ 157152,+ 34355,+ 7011,+ 26227,+ 5286,+ 40904,+ 311,+ 156895,+ 157151,+ 8469,+ 7342,+ 5468,+ 157152,+ 198,+ 156903,+ 156904+ ],+ "out": [+ 40904,+ 26312,+ 39765,+ 290,+ 18565,+ 4334,+ 2589,+ 424,+ 22573,+ 84624,+ 447,+ 45661,+ 23183,+ 3878+ ],+ "text": "珠海是中国改革开放的窗口城市,以“浪漫之都”闻名,拥有博"+ }+}\ No newline at end of filediff --git a/scripts/convert_adapters_legacy.py b/scripts/convert_adapters_legacy.pynew file mode 100644index 0000000..e8a01de--- /dev/null+++ b/scripts/convert_adapters_legacy.py@@ -0,0 +1,136 @@+#!/usr/bin/env python3+"""One-shot legacy adapter converter: npz -> safetensors.++The training pipelines exported adapters as ``.npz`` (qwen round-7 and+ling v7). edge0 consumes safetensors ONLY — this script converts each+legacy pair once into ``artifacts/`` (gitignored), writing the+conversion provenance (model, kind, K, r/alpha, source md5, format+version) into the safetensors ``__metadata__``. The npz sources are+NOT deleted (see the conversion note in README); they are only needed+to regenerate the artifacts.++Usage:+ python scripts/convert_adapters_legacy.py [--force]++Idempotent: existing artifacts are skipped unless ``--force``.+"""++from __future__ import annotations++import argparse+import hashlib+import json+import sys+from datetime import datetime, timezone+from pathlib import Path++import numpy as np++ROOT = Path(__file__).resolve().parents[1]+ARTIFACTS = ROOT / "artifacts"++# (npz path, tier, kind) — every legacy source this script understands.+SOURCES = [+ ("/Users/linyu/Documents/qwen35-v7-deploy/pregate_qwen35_v7_round7.npz",+ "edge0-35b", "prerouter"),+ ("/Users/linyu/Documents/qwen35-v7-deploy/lora_qwen35_v7_round7.npz",+ "edge0-35b", "lora"),+ ("/Users/linyu/Documents/ling-mlx-server/v7-deploy/pregate_v7_fp16.npz",+ "edge0-10b", "prerouter"),+ ("/Users/linyu/Documents/ling-mlx-server/v7-deploy/lora_v7_fp16.npz",+ "edge0-10b", "lora"),+]+++def md5(path: Path) -> str:+ h = hashlib.md5()+ with open(path, "rb") as f:+ for chunk in iter(lambda: f.read(1 << 20), b""):+ h.update(chunk)+ return h.hexdigest()+++def rewrite_key(key: str, tier: str, kind: str) -> str:+ """Map a legacy npz key to the edge0 artifact key.++ prerouter heads are normalized to ``layers.<N>.<part>.weight``;+ qwen lora gains the ``language_model.`` prefix (the checkpoint+ namespace the adapter is applied into).+ """+ if kind == "prerouter":+ # qwen round-7 npz already uses layers.N.*; ling v7 uses pregate.N.*+ parts = key.split(".")+ if parts[0] == "pregate":+ return f"layers.{parts[1]}.{parts[2]}.{parts[3]}"+ return key+ if kind == "lora" and tier == "edge0-35b" and key.startswith("model.layers."):+ return "language_model." + key+ return key+++def convert(source: Path, tier: str, kind: str, force: bool = False) -> Path:+ name = {+ ("edge0-35b", "prerouter"): "prerouter_edge0_35b_k4.safetensors",+ ("edge0-35b", "lora"): "lora_edge0_35b_k4.safetensors",+ ("edge0-10b", "prerouter"): "prerouter_edge0_10b.safetensors",+ ("edge0-10b", "lora"): "lora_edge0_10b.safetensors",+ }[(tier, kind)]+ out = ARTIFACTS / name+ if out.exists() and not force:+ print(f"skip {out.name} (exists)")+ return out++ data = np.load(source)+ tensors = {}+ for key in data.files:+ tensors[rewrite_key(key, tier, kind)] = data[key]++ # provenance: rank from the A matrix, alpha from the training run+ # (32 for both shipped pairs), K from the tier profile.+ meta = {+ "model": tier,+ "kind": kind,+ "K": "4" if tier == "edge0-35b" else "8",+ "r": "16", "alpha": "32",+ "source": str(source),+ "source_md5": md5(source),+ "converted": datetime.now(timezone.utc).isoformat(timespec="seconds"),+ "format_version": "1",+ }+ if kind == "prerouter":+ owners = sorted({int(k.split(".")[1])+ for k in tensors if k.startswith("layers.")})+ meta["owners"] = json.dumps(owners)++ from safetensors.numpy import save_file as save_np+ save_np(tensors, str(out), metadata={"__metadata__": json.dumps(meta)})+ print(f"write {out.name} ({len(tensors)} tensors, "+ f"{out.stat().st_size >> 20} MiB)")+ return out+++def main(argv: list[str] | None = None) -> int:+ ap = argparse.ArgumentParser(description=__doc__)+ ap.add_argument("--force", action="store_true",+ help="reconvert even if the artifact exists")+ args = ap.parse_args(argv)++ ARTIFACTS.mkdir(parents=True, exist_ok=True)+ missing = [s[0] for s in SOURCES if not Path(s[0]).is_file()]+ if missing:+ print("missing legacy sources (skip their conversion):")+ for m in missing:+ print(" -", m)+ for path, tier, kind in SOURCES:+ if not Path(path).is_file():+ continue+ try:+ convert(Path(path), tier, kind, force=args.force)+ except Exception as exc: # noqa: BLE001 — report and continue+ print(f"FAIL {Path(path).name}: {exc}", file=sys.stderr)+ return 1+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/e2e_smoke.py b/scripts/e2e_smoke.pynew file mode 100644index 0000000..216abe2--- /dev/null+++ b/scripts/e2e_smoke.py@@ -0,0 +1,83 @@+"""End-to-end smoke: real checkpoints, real adapters.++Runs both shipped tiers through the full prerouter + lora + staged-decode+path and cross-checks staged vs exact math by comparing GREEDY token+sequences (deterministic, so any staged/exact divergence shows up as a+different sequence).++Usage:+ .venv/bin/python scripts/e2e_smoke.py [--qwen-dir DIR] [--ling-dir DIR]+"""++from __future__ import annotations++import argparse+import sys+import time++from edge0 import AutoEngine+from edge0.backends import io+from edge0.config import GenerationConfig+from edge0.streaming.options import LayerOptions++QWEN_DEFAULT = "/Users/linyu/Documents/qwen35-v7-deploy/model"+LING_DEFAULT = "/Users/linyu/Documents/ling-mlx-server/v7-deploy/model"+++def _prompt_ids(model_dir: str, text: str) -> list[int]:+ tok = io.load_tokenizer(model_dir)+ return list(tok(text)["input_ids"])+++def run(name: str, model_dir: str, prompt_ids: list[int], max_new: int,+ options: LayerOptions | None = None, label: str = ""):+ t0 = time.time()+ kwargs = {"name": name, "model_dir": model_dir}+ if options is not None:+ kwargs["options"] = options # None would override the tier default+ eng = AutoEngine.from_pretrained(**kwargs)+ t_load = time.time() - t0+ t0 = time.time()+ out = eng.generate(+ prompt_ids,+ GenerationConfig(temperature=0.0, top_k=None, top_p=None,+ max_new_tokens=max_new),+ max_new_tokens=max_new)+ t_gen = time.time() - t0+ eng.close()+ print(f"[{name}{label}] load={t_load:.1f}s gen={t_gen:.1f}s "+ f"({max_new / t_gen:.1f} tok/s) tokens={out}")+ return out+++def main() -> int:+ ap = argparse.ArgumentParser()+ ap.add_argument("--qwen-dir", default=QWEN_DEFAULT)+ ap.add_argument("--ling-dir", default=LING_DEFAULT)+ ap.add_argument("--max-new", type=int, default=16)+ args = ap.parse_args()++ qwen_prompt = _prompt_ids(args.qwen_dir, "Hello!")+ ling_prompt = _prompt_ids(args.ling_dir, "你好")++ staged = run("edge0-35b", args.qwen_dir, qwen_prompt, args.max_new,+ label="[staged]")+ exact = run("edge0-35b", args.qwen_dir, qwen_prompt, args.max_new,+ options=LayerOptions(staged=False, staged_replace=False,+ full_layer_prefill=False,+ hot_per_layer=0, use_compile=False,+ history_prefetch=False),+ label="[exact]")+ if staged != exact:+ print("MISMATCH: staged vs exact greedy sequences differ "+ f"(staged={staged} exact={exact})")+ return 1+ print("qwen staged == exact greedy: OK")++ run("edge0-10b", args.ling_dir, ling_prompt, args.max_new, label="[staged]")+ print("edge0 e2e smoke OK")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/verify_alignment.py b/scripts/verify_alignment.pynew file mode 100644index 0000000..27f5355--- /dev/null+++ b/scripts/verify_alignment.py@@ -0,0 +1,170 @@+#!/usr/bin/env python+"""Adapter alignment check: correct demos (round8 / r3traces) vs edge0 artifacts.++Runs the SAME prompt through the correct demo engine (production env from+start_server.sh, with the NEW adapters: qwen round8, ling r3traces) and+through edge0 (artifacts/*.safetensors), both greedy, and writes both token+sequences + decoded text to one JSON result file. Identical out-ids =>+adapters mount identically.++Usage:+ python verify_alignment.py demo qwen+ python verify_alignment.py edge0 qwen+ python verify_alignment.py demo ling+ python verify_alignment.py edge0 ling++Result file: scripts/alignment_results.json (each run merges its entry).+"""+import argparse+import json+import os+import sys++N = 14 # max new tokens (greedy)+PROMPTS = {+ "qwen": "Hello! Write one short sentence about Zhuhai.",+ "ling": "你好,用一句话介绍珠海。",+}+QMODEL = "/Users/linyu/Documents/qwen35-v7-deploy/model"+LMODEL = "/Users/linyu/Documents/ling-mlx-server/v7-deploy/model"+QDIR = "/Users/linyu/Documents/qwen35-v7-deploy"+LDIR = "/Users/linyu/Documents/ling-mlx-server/v7-deploy"+RESULT = os.path.join(os.path.dirname(os.path.abspath(__file__)),+ "alignment_results.json")+++def save(tag, payload):+ data = {}+ if os.path.exists(RESULT):+ with open(RESULT) as f:+ data = json.load(f)+ data[tag] = payload+ with open(RESULT, "w") as f:+ json.dump(data, f, ensure_ascii=False, indent=2)+ print("saved", tag)+++def qwen_demo():+ os.environ.update(+ MODEL_DIR=QMODEL,+ LORA_ADAPTERS=QDIR + "/lora_qwen35_v7_round8.npz",+ LORA_R="16", LORA_ALPHA="32",+ PREGATE_NPZ=QDIR + "/pregate_qwen35_v7_round8.npz",+ QWEN_PG_START="7", QWEN_PG_HIDDEN="512",+ QWEN_STAGED="1", QWEN_STAGED_TOP_K="4", QWEN_STAGED_SLOTS="4",+ QWEN_STAGED_REPLACE="0", QWEN_PREGATE_V7="0",+ QWEN_HISTORY_PREFETCH="0", QWEN_INTRA_STAGE="0", QWEN_ASM_CACHE="0",+ )+ sys.path.insert(0, QDIR)+ import serve # noqa: E402 (ENGINE built in main(); build it here)+ from mlx_lm.utils import load_tokenizer # noqa: E402+ tok = load_tokenizer(QMODEL)+ serve.ENGINE = serve.QwenStreamingEngine(+ QMODEL, cache_slots=900, hot_per_layer=0)+ serve.ENGINE.load_tokenizer(tok)+ eng = serve.ENGINE+ tok = eng.tokenizer()+ messages = [{"role": "user", "content": PROMPTS["qwen"]}]+ ids = serve.encode_chat(tok, messages)+ eng.prefill(ids)+ logits = eng.next_logits()+ cur = int(serve.mx.argmax(logits).item())+ out = [cur]+ for _ in range(N - 1):+ logits = eng.step(out[-1])+ serve.mx.eval(logits)+ cur = int(serve.sample(logits, temperature=0.0, top_k=1, top_p=1.0,+ repetition_penalty=1.0, history=ids + out))+ out.append(cur)+ save("demo_qwen", {"prompt_ids": ids, "out": out, "text": tok.decode(out)})+++def edge0_qwen():+ from edge0 import AutoEngine # noqa: E402+ from edge0.config import GenerationConfig # noqa: E402+ eng = AutoEngine.from_pretrained(QMODEL)+ tok = eng._tok+ messages = [{"role": "user", "content": PROMPTS["qwen"]}]+ text = tok.apply_chat_template(+ messages, tokenize=False, add_generation_prompt=True)+ ids = list(tok(text)["input_ids"])+ out = eng.generate(ids, GenerationConfig(temperature=0.0, top_k=1,+ top_p=1.0,+ repetition_penalty=1.0),+ max_new_tokens=N)+ save("edge0_qwen", {"prompt_ids": ids, "out": out, "text": tok.decode(out)})+ eng.close()+++def ling_demo():+ os.environ.update(+ CHECKPOINT_DIR=LMODEL,+ PREGATE_HYBRID="1", PREGATE_INTRA="0", PREGATE_START_LAYER="1",+ PREGATE_FEATURE_TOPK="executed", PREGATE_DTYPE="fp16",+ PREGATE_NPZ=LDIR + "/pregate_r3traces.npz",+ LORA_ADAPTERS=LDIR + "/lora_r3traces.npz",+ THINK_MODE="0", REPETITION_PENALTY="1.0",+ STAGED_DECODE="0", STAGED_SYNC="0",+ STAGED_SLOTS="8", STAGED_LAYERS="1-23", EXPERT_CACHE_SLOTS="64",+ PREFETCH_ENABLE="0", PREFILL_FULL_LAYER="1", PREFILL_PIN="1",+ LING_PREWARM="1", MLX_COMPILE_MOE="1", LING_PREFILL_CHUNK="2048",+ )+ sys.path.insert(0, LDIR)+ import server # noqa: E402 (engine built in main(); build it here)+ server.engine = server.LingStreamingEngine(+ LMODEL, think=False, prefill_chunk=512)+ eng = server.engine+ msgs = [{"role": "user", "content": PROMPTS["ling"]}]+ ids = eng.encode_chat(msgs, think=False)+ eng.prefill_prompt(ids, chunk_size=512)+ out = []+ cur = int(server.mx.argmax(eng.next_logits(), axis=-1).item())+ out.append(cur)+ for _ in range(N - 1):+ logits = eng.step(out[-1])+ server.mx.eval(logits)+ cur = int(server.mx.argmax(logits, axis=-1).item())+ out.append(cur)+ if cur == 156895:+ break+ save("demo_ling", {"prompt_ids": ids, "out": out,+ "text": eng.decode(out)})+++def edge0_ling():+ from edge0 import AutoEngine # noqa: E402+ from edge0.config import GenerationConfig # noqa: E402+ eng = AutoEngine.from_pretrained(LMODEL)+ tok = eng._tok+ messages = [{"role": "user", "content": PROMPTS["ling"]}]+ text = tok.apply_chat_template(+ messages, tokenize=False, add_generation_prompt=True,+ enable_thinking=False)+ ids = list(tok(text)["input_ids"])+ eng.prefill(ids, chunk_size=512)+ out = []+ cur = int(eng.next_logits().argmax().item())+ out.append(cur)+ for _ in range(N - 1):+ logits = eng.step(out[-1])+ cur = int(logits.argmax().item())+ out.append(cur)+ if cur == 156895:+ break+ save("edge0_ling", {"prompt_ids": ids, "out": out,+ "text": tok.decode(out)})+ eng.close()+++def main():+ ap = argparse.ArgumentParser(description=__doc__)+ ap.add_argument("side", choices=["demo", "edge0"])+ ap.add_argument("model", choices=["qwen", "ling"])+ args = ap.parse_args()+ fn = {"demo": {"qwen": qwen_demo, "ling": ling_demo},+ "edge0": {"qwen": edge0_qwen, "ling": edge0_ling}}[args.side][args.model]+ fn()+++if __name__ == "__main__":+ main()diff --git a/scripts/zhuhai_example.py b/scripts/zhuhai_example.pynew file mode 100644index 0000000..77f0ebd--- /dev/null+++ b/scripts/zhuhai_example.py@@ -0,0 +1,90 @@+"""Complete end-to-end example: generate an English introduction to+Zhuhai with edge0-35b (prerouter + LoRA + SSD offload, default tier+profile — staged decode on, K=4).++Usage:+ .venv/bin/python scripts/zhuhai_example.py [--max-new N] [--temp T]++The example walks through the full pipeline a user of the framework+would hit:++1. load the tokenizer and the engine (real checkpoint + adapters),+2. tokenize a prompt,+3. prefill (whole-layer load-drop, hot pins),+4. greedy preview of the first token,+5. sampled generation with the tier's default GenerationConfig,+6. decode back to text.+"""++from __future__ import annotations++import argparse+import sys+import time++from edge0 import AutoEngine+from edge0.backends import io+from edge0.config import GenerationConfig+++PROMPT = (+ "Write a short introduction to Zhuhai, a city in southern China, "+ "in English. Keep it to 3-4 sentences."+)+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__)+ ap.add_argument("--max-new", type=int, default=64)+ ap.add_argument("--temp", type=float, default=0.7)+ ap.add_argument("--thinking", action="store_true",+ help="enable the <think> block (enable_thinking=True)")+ args = ap.parse_args()++ model_dir = "/Users/linyu/Documents/qwen35-v7-deploy/model"+ print(f"[1] loading tokenizer + edge0-35b from {model_dir}", flush=True)+ tok = io.load_tokenizer(model_dir)+ t0 = time.time()+ eng = AutoEngine.from_pretrained(name="edge0-35b", model_dir=model_dir)+ print(f"[2] engine built in {time.time() - t0:.1f}s", flush=True)++ ids = list(tok(PROMPT)["input_ids"])+ print(f"[3] prompt: {len(ids)} tokens: {ids[:12]} ...", flush=True)++ # Deployment parity (bench_v7.py / serve.py): build the prompt with+ # the chat template. enable_thinking=False closes an empty <think>+ # block so the model answers directly; --thinking leaves the block+ # open so the model emits reasoning first, then </think> + the answer.+ chat = tok.apply_chat_template(+ [{"role": "user", "content": PROMPT}],+ add_generation_prompt=True, tokenize=True,+ enable_thinking=args.thinking)+ chat = list(chat.input_ids)+ print(f"[3b] chat template: {len(chat)} tokens", flush=True)++ # Greedy preview of the first token (deterministic).+ t0 = time.time()+ eng.prefill(chat)+ l0 = eng.next_logits()+ first = int(l0.argmax().item())+ print(f"[4] prefill in {time.time() - t0:.1f}s; first greedy token "+ f"{first} = {tok.decode([first])!r}", flush=True)+ eng.reset()++ # Full generation with the tier default sampler (temp 0.7, top_p 0.95).+ gc = GenerationConfig(temperature=args.temp, max_new_tokens=args.max_new)+ t0 = time.time()+ out = eng.generate(chat, gc, max_new_tokens=args.max_new)+ dt = time.time() - t0+ text = tok.decode(out)+ print(f"[5] generated {len(out)} tokens in {dt:.1f}s "+ f"({len(out) / dt:.1f} tok/s)", flush=True)+ print("=" * 60, flush=True)+ print(text, flush=True)+ print("=" * 60, flush=True)+ eng.close()+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/src/edge0/__init__.py b/src/edge0/__init__.pynew file mode 100644index 0000000..432e6ca--- /dev/null+++ b/src/edge0/__init__.py@@ -0,0 +1,35 @@+"""edge0: streaming MoE inference for Apple Silicon.++SSD expert offload + parallel LoRA + prerouter routing prediction, with a+transformers-style AutoModel registry and a backend-isolated design (all+MLX code lives under ``edge0.backends.mlx``; a CUDA backend can be added+as a sibling package implementing the same interface).++Public API::++ from edge0 import AutoConfig, AutoModel, AutoEngine++ cfg = AutoConfig.from_pretrained(model_dir)+ model = AutoModel.from_pretrained(model_dir, name="edge0-35b", ...)+ engine = AutoEngine.from_pretrained(model_dir, ...)+"""++from edge0 import registry # noqa: F401 (populates the model registry)++__version__ = "0.1.0"++__all__ = [+ "__version__",+ "AutoConfig",+ "AutoModel",+ "AutoEngine",+]+++def __getattr__(name: str):+ # Deferred imports so `import edge0` stays light and backend selection+ # (EDGE0_BACKEND) happens on first use.+ if name in ("AutoConfig", "AutoModel", "AutoEngine"):+ from edge0 import registry as _reg+ return getattr(_reg, name)+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")diff --git a/src/edge0/__main__.py b/src/edge0/__main__.pynew file mode 100644index 0000000..24576b1--- /dev/null+++ b/src/edge0/__main__.py@@ -0,0 +1,7 @@+"""``python -m edge0`` entry point (same surface as the ``edge0``+console script)."""++from edge0.cli import main++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/src/edge0/adapters/__init__.py b/src/edge0/adapters/__init__.pynew file mode 100644index 0000000..0cd99f4--- /dev/null+++ b/src/edge0/adapters/__init__.py@@ -0,0 +1,5 @@+"""Adapter installation (parallel LoRA + prerouter heads)."""++from edge0.adapters.lora import LoraLinear, install_lora++__all__ = ["LoraLinear", "install_lora"]diff --git a/src/edge0/adapters/lora.py b/src/edge0/adapters/lora.pynew file mode 100644index 0000000..93b5beb--- /dev/null+++ b/src/edge0/adapters/lora.py@@ -0,0 +1,125 @@+"""Parallel (unmerged) LoRA adapters, safetensors-native.++Port of the deployment's ``lora_apply.py`` scheme (both qwen and ling+tiers): each adapter target is wrapped as a *parallel delta path*::++ y = base(x) + scale * ((x @ A.T) @ B.T) # scale = alpha / r++The base weights (including 4bit quantized linears) stay byte-identical —+no dequant / merge / requant — and the LoRA lives only in tiny fp16 A/B+tensors. Mathematically identical to a merged weight.++Safetensors key naming (framework contract)::++ <target>.lora_A [r, in]+ <target>.lora_B [out, r]++where ``<target>`` is the module path inside the loaded model (e.g.+``model.layers.3.attention.q_proj`` or+``language_model.model.layers.0.linear_attn.in_proj_qkv``). The legacy+npz files used the same A/B naming, so conversion is a pure format+migration (see ``scripts/convert_adapters_legacy.py``).+"""++from __future__ import annotations++from edge0.backends import core+from edge0.backends import nn++from edge0.backends.mlx.io import load_safetensors+++class LoraLinear(nn.Module):+ """Wraps a Linear/QuantizedLinear and adds the low-rank delta."""++ def __init__(self, base: nn.Module, lora_a: core.array, lora_b: core.array,+ scale: float):+ super().__init__()+ self.base = base+ self.lora_A = lora_a # [r, in]+ self.lora_B = lora_b # [out, r]+ self.lora_scale = scale++ def __call__(self, x: core.array) -> core.array:+ y = self.base(x)+ ad = self.lora_A.dtype+ xd = x if x.dtype == ad else x.astype(ad)+ # delta = (B @ A) applied on x: ((x @ A.T) @ B.T), shape [..., out]+ d = (xd @ self.lora_A.T) @ self.lora_B.T+ return y + (self.lora_scale * d).astype(y.dtype)+++def _resolve(model: nn.Module, key: str):+ """Resolve ``model.layers.N.<...>`` or a bare dotted path to the+ module inside the loaded model."""+ parts = key.split(".")+ if len(parts) >= 3 and parts[0] == "model" and parts[1] == "layers":+ node = model.model.layers[int(parts[2])]+ rest = parts[3:]+ elif len(parts) >= 3 and parts[0] == "language_model" and parts[1] == "model":+ node = model.language_model.model.layers[int(parts[3])]+ rest = parts[4:]+ else:+ node = model+ rest = parts+ for p in rest:+ node = getattr(node, p)+ return node+++def install_lora(model: nn.Module, adapters_path: str,+ r: int = 16, alpha: float = 32.0,+ dtype=core.float16, strict: bool = True) -> dict:+ """Attach every LoRA pair in ``adapters_path`` (safetensors) as a+ parallel delta path.++ Returns ``{applied: [...], not_found: [...], skipped: [...],+ scale: float, dtype: str}``.+ """+ lora = load_safetensors(adapters_path)+ if not lora:+ raise ValueError("adapters has no lora tensors")++ # group by target key+ targets: dict[str, dict] = {}+ for k, v in lora.items():+ if k.endswith(".lora_A"):+ targets.setdefault(k[: -len(".lora_A")], {})["A"] = v+ elif k.endswith(".lora_B"):+ targets.setdefault(k[: -len(".lora_B")], {})["B"] = v+ else:+ raise ValueError(f"unexpected lora key: {k}")++ scale = alpha / r+ applied, not_found, skipped = [], [], []++ for key, d in sorted(targets.items()):+ if "A" not in d or "B" not in d:+ skipped.append((key, "missing A or B"))+ continue+ mod = _resolve(model, key)+ if mod is None:+ not_found.append(key)+ if strict:+ raise KeyError(f"lora target not found in MLX model: {key}")+ continue+ if not hasattr(mod, "weight") and not hasattr(mod, "base"):+ # not a linear layer we can wrap (e.g. RMSNorm / gate)+ skipped.append((key, type(mod).__name__))+ if strict:+ raise TypeError(+ f"lora target {key} is {type(mod).__name__}, not Linear")+ continue+ a = d["A"].astype(dtype)+ b = d["B"].astype(dtype)+ # re-bind the module attribute (e.g. attention.q_proj = LoraLinear)+ parts = key.split(".")+ owner = _resolve(model, ".".join(parts[:-1]))+ attr = parts[-1]+ setattr(owner, attr, LoraLinear(mod, a, b, scale))+ applied.append(key)++ print(f"[lora] applied={len(applied)} not_found={len(not_found)} "+ f"skipped={len(skipped)} scale={scale}", flush=True)+ return {"applied": applied, "not_found": not_found, "skipped": skipped,+ "scale": scale, "dtype": str(dtype)}diff --git a/src/edge0/attention/spec.py b/src/edge0/attention/spec.pynew file mode 100644index 0000000..88928e3--- /dev/null+++ b/src/edge0/attention/spec.py@@ -0,0 +1,71 @@+"""Attention taxonomy.++edge0 does not re-implement attention kernels: the vendored base model+implementations (``edge0.backends.mlx._impl``) carry the kernels (GQA /+GatedDeltaNet for edge0-35b, MLA / DeltaNet for edge0-10b). This module+describes attention so the framework can introspect a model — cache+sizing, layer roles, documentation, and future kernels — without knowing+the concrete implementation.++A new attention kind = a new ``AttentionKind`` member + an+``AttentionSpec`` description + a kernel implementation (in a vendored+base or a backend ``_impl`` module). See docs/attention.md.+"""++from __future__ import annotations++from dataclasses import dataclass+from enum import Enum+++class AttentionKind(Enum):+ """Attention / sequence-mixing families supported by edge0 models."""++ GQA = "gqa"+ """Grouped-query attention with softmax and KV cache."""++ MLA = "mla"+ """Multi-head latent attention (compressed KV)."""++ DELTANET = "deltanet"+ """DeltaNet linear attention (gated delta rule, no KV cache)."""++ GATED_DELTANET = "gated_deltanet"+ """GatedDeltaNet (gated delta rule with decay / linear attention)."""++ DENSE_MLP = "dense_mlp"+ """No sequence mixing (mlp-only layer)."""+++@dataclass(frozen=True)+class AttentionSpec:+ """Description of one layer's sequence-mixing block.++ Attributes:+ kind: attention family.+ layer_indices: 0-based layer indices covered by this spec.+ num_heads / num_kv_heads / head_dim: head geometry+ (GQA/MLA only; None otherwise).+ cache: whether the block maintains a KV-style cache+ (False for linear-attention layers).+ notes: free-form (e.g. hybrid schedules).+ """++ kind: AttentionKind+ layer_indices: tuple[int, ...]+ num_heads: int | None = None+ num_kv_heads: int | None = None+ head_dim: int | None = None+ cache: bool = True+ notes: str = ""++ def __repr__(self) -> str: # compact one-liner for logs/docs+ return (f"AttentionSpec({self.kind.value}, layers="+ f"{self.layer_indices[0]}..{self.layer_indices[-1]}, "+ f"cache={self.cache})")+++def summarize(specs: list[AttentionSpec]) -> str:+ """Human-readable one-line summary, e.g. for the CLI banner."""+ return ", ".join(+ f"{s.kind.value}x{len(s.layer_indices)}" for s in specs)diff --git a/src/edge0/backends/__init__.py b/src/edge0/backends/__init__.pynew file mode 100644index 0000000..71e662c--- /dev/null+++ b/src/edge0/backends/__init__.py@@ -0,0 +1,61 @@+"""Backend abstraction.++The framework core (engine, streaming, prerouter, adapters, model+adapters, server) imports ONLY the backend namespaces re-exported here::++ from edge0.backends import core, nn, io, quant++A backend is a directory under ``edge0/backends/<name>/`` exposing:++``backend.py``+ ``core`` — array ops namespace (matmul, softmax, argpartition, take,+ put_along_axis, eval, compile, ...)+ ``nn`` — module factory namespace (Module, Linear, RMSNorm, silu, ...)+ ``io`` — model/tokenizer/tensor-store loading+ ``quant`` — quantized gather kernels (``gather_qmm``)+ ``BackendImpl`` — small object with ``name`` and metadata++The MLX backend is the reference implementation. A future CUDA backend+implements the same surface with eager semantics; engine / prerouter /+server / model-adapter code is shared verbatim. The op list below is the+contract the framework currently exercises (grow it as models are added).++Contract (documented, enforced by tests/grep in CI):+ * The MLX package (``mlx`` / ``mlx_lm``) must never be imported+ outside ``edge0/backends/mlx/`` — framework code reaches the backend+ only through the namespaces re-exported here.+ * ``core`` must provide (for the MLX backend these are mlx.core names):+ array, zeros, eye, arange, full, expand_dims, squeeze, reshape,+ transpose, concatenate, stack, split, matmul, softmax, silu (via nn),+ sigmoid, erf, where, sum, cumsum, sort, topk, argpartition, take,+ take_along_axis, put_along_axis, astype, item, tolist, eval, compile,+ random.seed, random.categorical, and the dtypes float16 / float32 /+ bfloat16 / int32 / uint32 / int64.+ * ``nn`` must provide Module, Linear, RMSNorm, silu, gelu.+ * ``io`` must provide load_model, load_tokenizer, open_tensor_store.+ * ``quant`` must provide gather_qmm.+"""++import os++_BACKEND = os.environ.get("EDGE0_BACKEND", "mlx")++if _BACKEND == "mlx":+ from edge0.backends.mlx.backend import ( # noqa: F401+ BackendImpl,+ core,+ io,+ nn,+ quant,+ )+elif _BACKEND == "cuda":+ raise ImportError(+ "the CUDA backend is not implemented yet; set EDGE0_BACKEND=mlx "+ "(or unset it)")+else:+ raise ImportError(+ f"unknown backend {_BACKEND!r}; available: mlx")++backend = BackendImpl()++__all__ = ["backend", "core", "io", "nn", "quant"]diff --git a/src/edge0/backends/base.py b/src/edge0/backends/base.pynew file mode 100644index 0000000..0b209c2--- /dev/null+++ b/src/edge0/backends/base.py@@ -0,0 +1,48 @@+"""Tensor-store protocol: where adapter weights (LoRA, prerouter heads)+are loaded from.++The canonical format is safetensors (mmap-backed, lazy, with provenance+metadata). No other format is supported.+"""++from __future__ import annotations++from abc import ABC, abstractmethod+++class TensorStore(ABC):+ """Read-only view over a weights file (adapter artifacts).++ Implementations must be lazily opened and must not materialize the+ whole file into memory.+ """++ @abstractmethod+ def keys(self) -> list[str]:+ """Sorted tensor names in the store."""++ @abstractmethod+ def get(self, name: str):+ """Return one tensor as an array (backend-native).++ Raises KeyError for unknown names.+ """++ @abstractmethod+ def metadata(self) -> dict:+ """File-level provenance metadata (safetensors __metadata__)."""++ @property+ @abstractmethod+ def path(self) -> str:+ """Source file path (for error messages / logging)."""+++def open_tensor_store(path: str) -> TensorStore:+ """Open a tensor store by extension (safetensors)."""+ if path.endswith(".safetensors"):+ from edge0.backends import io # deferred: backend selection+ return io.SafeTensorsStore(path)+ raise ValueError(+ f"unsupported adapter format: {path!r} (only .safetensors is "+ f"supported)")diff --git a/src/edge0/backends/mlx/__init__.py b/src/edge0/backends/mlx/__init__.pynew file mode 100644index 0000000..e092433--- /dev/null+++ b/src/edge0/backends/mlx/__init__.py@@ -0,0 +1,9 @@+"""MLX backend — the reference backend implementation.++Everything that touches ``mlx`` lives under this package. Nothing else in+the framework may import mlx directly.+"""++from edge0.backends.mlx.backend import BackendImpl, core, io, nn, quant++__all__ = ["BackendImpl", "core", "io", "nn", "quant"]diff --git a/src/edge0/backends/mlx/_impl/__init__.py b/src/edge0/backends/mlx/_impl/__init__.pynew file mode 100644index 0000000..b9c0548--- /dev/null+++ b/src/edge0/backends/mlx/_impl/__init__.py@@ -0,0 +1,11 @@+"""Vendored base model implementations (licensing in NOTICE).++* ``qwen3_next.py`` — mlx-lm 0.31.0's Qwen3.5-MoE model, byte-identical+ to the upstream file except the import block, which was rewritten to+ absolute ``mlx_lm.models`` paths so the file can live outside the+ mlx_lm package. Pinned to mlx-lm==0.31.0 by pyproject.+* ``bailing_hybrid.py`` — the ling 3.0 backbone vendored from the+ production deployment, with the prerouter machinery renamed from the+ legacy identifiers (``*_gate*`` -> ``prerouter*``) and no other+ changes.+"""diff --git a/src/edge0/backends/mlx/_impl/bailing_hybrid.py b/src/edge0/backends/mlx/_impl/bailing_hybrid.pynew file mode 100644index 0000000..2f83282--- /dev/null+++ b/src/edge0/backends/mlx/_impl/bailing_hybrid.py@@ -0,0 +1,915 @@+# SPDX-License-Identifier: Apache-2.0+from __future__ import annotations++import os++"""Vendored inclusionAI Ling 3.0 backbone (``model_type: bailing_hybrid``).++**Why vendor:** Ling-3.0-tiny (7.9B total / 1.3B active, MIT) landed with+day-0 SGLang/vLLM support and no MLX story beyond an unmerged mlx-lm PR:++- mlx-lm: no ``bailing_hybrid`` (checked 0.31.3 + upstream main,+ 2026-08-10); PR ml-explore/mlx-lm#1227 (Ling-2.6-era shape) open and+ unreviewed since 2026-07-06 — the same situation ``hy_v3`` was vendored+ out of.+- transformers: no native port; the HF repo ships remote code that+ imports ``fla`` (triton) at module level, so it cannot even run on+ macOS.++One module serves the whole family: Ling-3.0-tiny, Ling-3.0-flash and+Ling-2.6-flash all declare ``bailing_hybrid``.++**References (math verified against ALL of):**++- HF remote code ``modeling_bailing_moe_v3.py`` — authoritative+- fla kernels (``fla/ops/kda``) — exact ``safe_gate``/``lower_bound``+ semantics (see ``_kda_gate``)+- mlx-lm ``kimi_linear.py`` — MLX KDA idiom (KDA is Kimi's design;+ Ling 3.0 reuses it with a safe-gate variant); ``ShortConv1d`` and the+ group expert select are adapted from it+- mlx-lm PR #1227 — Ling-2.6 MLX port (cross-check)++Architecture (Ling-3.0-tiny values):++- 24 layers in ``[KDA ×3, MLA]`` groups (``layer_group_size`` 4; the+ trailing remainder layers are MLA); layer 0 is a dense MLP+ (``first_k_dense_replace`` 1), all later layers are sparse MoE+- KDA (Kimi Delta Attention, linear): q/k/v projections through causal+ short convolutions (kernel 4, silu), per-head decay gate with the+ SAFE-GATE law ``g = lower_bound * sigmoid(exp(A_log) * (f(x) ++ dt_bias))`` (fla ``USE_LOWER_BOUND`` branch — NOT the softplus law+ mlx-lm's ``compute_g`` implements), delta-rule state update, gated+ RMSNorm output+- MLA (DeepSeek-style): q LoRA 256 / kv LoRA 512, 128 nope + 64 rope+ dims, interleaved RoPE theta 6e6, plus a V3-only HEAD-WISE output+ gate (``g_proj``: hidden → n_heads, output scaled by sigmoid); the+ output projection is named ``dense`` in the checkpoint+- MoE: 128 routed experts top-8 + 1 shared, sigmoid scores with+ ``noaux_tc`` expert-bias selection, 8 groups pick 4+ (``topk_group``), normalized top-k probs scaled by 2.5+- untied ``lm_head``; embeddings under ``model.word_embeddings``++**Registration:** installed as ``sys.modules["mlx_lm.models.bailing_hybrid"]``+by ``vllm_mlx.utils.tokenizer._register_vendored_archs`` (same trick as+``deepseek_v4`` / ``hy_v3`` / ``muse_glimmer``), with a ``find_spec``+probe that defers to native mlx-lm support the moment upstream ships it.++**Sync policy:** when mlx-lm merges #1227's successor with V3 support,+diff and delete this file.+"""++from dataclasses import dataclass+from typing import Any++import mlx.core as mx+import mlx.nn as nn++# Install the MLX hardware compatibility shim before importing any mlx-lm+# module. mlx-lm captures its default stream during package import, which is+# unsafe on the M5 single-stream path; every vendored model follows this+# ordering contract.+from mlx_lm.models.base import (+ BaseModelArgs,+ create_attention_mask,+ scaled_dot_product_attention,+)+from mlx_lm.models.cache import ArraysCache, KVCache+from mlx_lm.models.switch_layers import SwitchGLU+++def _rope_interleave_torch(+ x: mx.array, positions: mx.array, theta: float+) -> mx.array:+ """Upstream ``apply_rotary_pos_emb_interleave``, verbatim from+ ``modeling_bailing_moe_v3`` (transformers 4.x ``default`` RoPE, which is+ what ``rope_scaling: null`` means for Ling-3.0-tiny):++ freqs = 1 / theta ** (arange(0, d, 2) / d)+ emb = cat((pos @ freqs, pos @ freqs))+ qi = q.view(d//2, 2).transpose(-1, -2).reshape(d)+ out = qi * cos + rotate_half(qi) * sin++ ``positions`` are absolute token ids (arange(L) + cache offset), matching+ torch's ``position_ids``/``cache_position``. We cannot use+ ``mx.fast.rope`` for this: its ``offset`` is a single absolute position+ (vector must have 1 element), so a prefill with offset=0 silently skips+ rotation. Verified against a float64 numpy replica of the upstream+ formula (rel ~1e-7, float32 rounding only).+ """+ B, H, L, D = x.shape+ freqs = 1.0 / (theta ** (mx.arange(0, D, 2, dtype=mx.float32) / D))+ angles = mx.outer(positions.astype(mx.float32), freqs)+ emb = mx.concatenate([angles, angles], axis=-1)+ cos = mx.cos(emb)[None, None, :, :]+ sin = mx.sin(emb)[None, None, :, :]+ xi = x.reshape(B, H, L, D // 2, 2).transpose(0, 1, 2, 4, 3).reshape(+ B, H, L, D+ )+ h = D // 2+ rh = mx.concatenate([-xi[..., h:], xi[..., :h]], axis=-1)+ return xi * cos + rh * sin+++@dataclass+class ModelArgs(BaseModelArgs):+ model_type: str = "bailing_hybrid"+ hidden_size: int = 1536+ num_hidden_layers: int = 24+ intermediate_size: int = 4608+ num_attention_heads: int = 16+ num_key_value_heads: int = 16+ head_dim: int = 128+ rms_norm_eps: float = 1e-6+ vocab_size: int = 157184+ max_position_embeddings: int = 131072+ tie_word_embeddings: bool = False+ # ---- layer schedule -------------------------------------------------+ layer_group_size: int = 4+ first_k_dense_replace: int = 1+ # ---- KDA (linear attention) ----------------------------------------+ short_conv_kernel_size: int = 4+ no_kda_lora: bool = True+ kda_safe_gate: bool = True+ kda_lower_bound: float = -5.0+ # ---- MLA -----------------------------------------------------------+ q_lora_rank: int | None = 256+ kv_lora_rank: int = 512+ qk_nope_head_dim: int = 128+ qk_rope_head_dim: int = 64+ v_head_dim: int = 128+ rope_theta: float = 6000000.0+ rope_interleave: bool = True+ rope_scaling: dict | None = None+ use_qkv_bias: bool = False+ gated_attention_proj_granularity_type: str | None = "head_wise"+ # ---- MoE -----------------------------------------------------------+ num_experts: int = 128+ num_experts_per_tok: int = 8+ num_shared_experts: int = 1+ moe_intermediate_size: int = 512+ moe_shared_expert_intermediate_size: int = 512+ n_group: int = 8+ topk_group: int = 4+ norm_topk_prob: bool = True+ routed_scaling_factor: float = 2.5+ moe_router_enable_expert_bias: bool = True+ # ---- pre-gated MoE (hybrid experiment) -----------------------------+ prerouter_enabled: bool = False+ prerouter_start_layer: int = 7+ prerouter_hidden: int = 512++ @property+ def qk_head_dim(self) -> int:+ return self.qk_nope_head_dim + self.qk_rope_head_dim++ def is_mla_layer(self, idx: int) -> bool:+ """Softmax (MLA) layers sit last in each group; any remainder+ layers past the final full group are MLA too (torch ref+ ``BailingMoeV3DecoderLayer.__init__``)."""+ g = self.layer_group_size+ full = self.num_hidden_layers // g * g+ return (idx + 1) % g == 0 or idx >= full+++class ShortConv1d(nn.Module):+ """Causal depthwise conv with silu and a rolling cache.++ Adapted from mlx-lm ``kimi_linear.ShortConv1d`` (same wire: the+ checkpoint stores ``*_conv1d.weight`` of shape [channels, ksize, 1]).+ """++ def __init__(self, channels: int, kernel_size: int):+ super().__init__()+ self.kernel_size = kernel_size+ self.conv = nn.Conv1d(+ channels, channels, kernel_size, groups=channels, bias=False+ )++ def __call__(+ self,+ x: mx.array,+ state: mx.array | None = None,+ ) -> tuple[mx.array, mx.array]:+ B, T, C = x.shape+ if state is None:+ state = mx.zeros((B, self.kernel_size - 1, C), dtype=x.dtype)+ conv_input = mx.concatenate([state, x], axis=1)+ out = nn.silu(self.conv(conv_input))+ if self.kernel_size == 1:+ # ``-(k-1)`` would be ``-0`` and retain the WHOLE input,+ # growing the cache every step (codex r3 #3).+ new_state = conv_input[:, :0, :]+ else:+ new_state = conv_input[:, -(self.kernel_size - 1) :, :]+ return out, new_state+++def _kda_gate(+ f: mx.array,+ A_log: mx.array, # noqa: N803 — the checkpoint's parameter name+ dt_bias: mx.array,+ *,+ safe_gate: bool,+ lower_bound: float,+) -> mx.array:+ """Per-channel log-decay gate.++ fla ``fused_recurrent_kda_fwd_kernel`` (L166-171):++ g_pre = f + dt_bias+ USE_LOWER_BOUND: g = lower_bound * sigmoid(exp(A_log) * g_pre)+ else: g = -exp(A_log) * softplus(g_pre)++ Ling 3.0 ships ``kda_safe_gate=True`` / ``kda_lower_bound=-5`` — the+ sigmoid law, clamping the log-decay into (lower_bound, 0). Computed+ in float32 like the kernel.+ """+ f = f.astype(mx.float32) + dt_bias.astype(mx.float32)+ a = mx.exp(A_log.astype(mx.float32))+ if safe_gate:+ return lower_bound * mx.sigmoid(a[..., None] * f)+ return -a[..., None] * nn.softplus(f)+++def _kda_update(+ q: mx.array,+ k: mx.array,+ v: mx.array,+ g_log: mx.array,+ beta: mx.array,+ state: mx.array | None,+) -> tuple[mx.array, mx.array]:+ """Run the KDA delta-rule recurrence via mlx-lm's fused gated-delta+ path (the same Metal kernel / ops pair qwen3-next and kimi_linear+ ship on), with our precomputed safe-gate decay.++ ``gated_delta_ops``/``gated_delta_kernel`` take the decay as a+ MULTIPLIER (``compute_g`` returns ``exp(...)``), vectorized per+ channel ``[B, T, H, Dk]`` — so the log-space safe gate is+ exponentiated here. A per-token Python loop over the full prompt+ (the naive fla reference) is prohibitively slow at 131K context+ (codex r1 #1); the kernel path chunks the recurrence on Metal.+ """+ from mlx_lm.models.gated_delta import gated_delta_kernel, gated_delta_ops++ g = mx.exp(g_log)+ if state is None:+ B = q.shape[0]+ state = mx.zeros((B, v.shape[-2], v.shape[-1], k.shape[-1]), dtype=mx.float32)+ # The Metal kernel tiles Dk in 32-lane strips (``n_per_t = Dk/32``);+ # head dims below 32 would generate a zero-length array and fail the+ # metallib build. Production checkpoints use head_dim 128; tiny test+ # configs take the ops path.+ use_kernel = (+ gated_delta_kernel is not None+ and k.shape[-1] % 32 == 0+ and mx.default_device() == mx.gpu+ and mx.metal.is_available()+ )+ fn = gated_delta_kernel if use_kernel else gated_delta_ops+ return fn(q, k, v, g, beta, state, None)+++class BailingKDA(nn.Module):+ """Kimi Delta Attention with the Ling V3 safe gate."""++ def __init__(self, args: ModelArgs, layer_idx: int):+ super().__init__()+ self.num_heads = args.num_attention_heads+ self.head_dim = args.head_dim+ self.proj_dim = self.num_heads * self.head_dim+ self.conv_kernel = args.short_conv_kernel_size+ self.safe_gate = args.kda_safe_gate+ self.lower_bound = float(args.kda_lower_bound)+ self.no_kda_lora = args.no_kda_lora+ self.scale = float(self.head_dim) ** -0.5++ hidden = args.hidden_size+ self.q_proj = nn.Linear(hidden, self.proj_dim, bias=False)+ self.k_proj = nn.Linear(hidden, self.proj_dim, bias=False)+ self.v_proj = nn.Linear(hidden, self.proj_dim, bias=False)+ self.q_conv1d = ShortConv1d(self.proj_dim, self.conv_kernel)+ self.k_conv1d = ShortConv1d(self.proj_dim, self.conv_kernel)+ self.v_conv1d = ShortConv1d(self.proj_dim, self.conv_kernel)++ if self.no_kda_lora:+ self.f_proj = nn.Linear(hidden, self.proj_dim, bias=False)+ self.g_proj = nn.Linear(hidden, self.proj_dim, bias=False)+ else:+ # Ling-2.6 / flash variants use LoRA pairs. The bottleneck is+ # head_dim BY DESIGN — the authoritative reference hard-codes+ # it the same way (modeling_bailing_moe_v3.BailingMoeV3+ # KimiDeltaAttention: ``f_a_proj = Linear(hidden_size,+ # head_dim)``) and no bailing config revision publishes a+ # separate f/g LoRA-rank field to wire.+ self.f_a_proj = nn.Linear(hidden, self.head_dim, bias=False)+ self.f_b_proj = nn.Linear(self.head_dim, self.proj_dim, bias=False)+ self.g_a_proj = nn.Linear(hidden, self.head_dim, bias=False)+ self.g_b_proj = nn.Linear(self.head_dim, self.proj_dim, bias=False)++ self.b_proj = nn.Linear(hidden, self.num_heads, bias=False)+ self.A_log = mx.zeros((self.num_heads,))+ self.dt_bias = mx.zeros((self.proj_dim,))+ self.o_norm = nn.RMSNorm(self.head_dim, eps=args.rms_norm_eps)+ self.o_proj = nn.Linear(self.proj_dim, hidden, bias=False)++ def __call__(+ self,+ x: mx.array,+ mask: mx.array | None = None,+ cache: Any | None = None,+ ) -> mx.array:+ B, T, _ = x.shape+ dtype = x.dtype++ if cache is not None:+ q_state, k_state, v_state, ssm_state = cache+ else:+ q_state = k_state = v_state = ssm_state = None++ q_conv, q_state = self.q_conv1d(self.q_proj(x), q_state)+ k_conv, k_state = self.k_conv1d(self.k_proj(x), k_state)+ v_conv, v_state = self.v_conv1d(self.v_proj(x), v_state)+ if cache is not None:+ cache[0] = q_state+ cache[1] = k_state+ cache[2] = v_state++ q = q_conv.reshape(B, T, self.num_heads, self.head_dim)+ k = k_conv.reshape(B, T, self.num_heads, self.head_dim)+ v = v_conv.reshape(B, T, self.num_heads, self.head_dim)++ # fla applies L2-norm to q/k in-kernel plus scale=d^-0.5. Use the+ # exact l2norm form (x / (||x|| + eps)) so the eps placement+ # matches the kernel bit-for-bit; the kimi_linear rms_norm+ # folding trick puts eps under the sqrt and costs ~2e-4 parity.+ qf = q.astype(mx.float32)+ kf = k.astype(mx.float32)+ q = self.scale * qf / (mx.linalg.norm(qf, axis=-1, keepdims=True) + 1e-6)+ k = kf / (mx.linalg.norm(kf, axis=-1, keepdims=True) + 1e-6)++ if self.no_kda_lora:+ f = self.f_proj(x)+ gate = self.g_proj(x)+ else:+ f = self.f_b_proj(self.f_a_proj(x))+ gate = self.g_b_proj(self.g_a_proj(x))+ f = f.reshape(B, T, self.num_heads, self.head_dim)+ g = _kda_gate(+ f,+ self.A_log,+ self.dt_bias.reshape(self.num_heads, self.head_dim),+ safe_gate=self.safe_gate,+ lower_bound=self.lower_bound,+ )+ beta = mx.sigmoid(self.b_proj(x).astype(mx.float32))++ out, ssm_state = _kda_update(q, k, v, g, beta, ssm_state)+ if cache is not None:+ cache[3] = ssm_state++ gate = gate.reshape(B, T, self.num_heads, self.head_dim)+ out = self.o_norm(out.astype(dtype)) * mx.sigmoid(gate)+ return self.o_proj(out.reshape(B, T, -1))+++class BailingMLA(nn.Module):+ """DeepSeek-style MLA plus the V3 head-wise output gate."""++ def __init__(self, args: ModelArgs):+ super().__init__()+ self.num_heads = args.num_attention_heads+ self.qk_nope_head_dim = args.qk_nope_head_dim+ self.qk_rope_head_dim = args.qk_rope_head_dim+ self.qk_head_dim = args.qk_head_dim+ self.v_head_dim = args.v_head_dim+ self.kv_lora_rank = args.kv_lora_rank+ self.q_lora_rank = args.q_lora_rank+ self.scale = self.qk_head_dim**-0.5+ self.gate_kind = args.gated_attention_proj_granularity_type++ hidden = args.hidden_size+ bias = args.use_qkv_bias+ if self.q_lora_rank is None:+ # bias=False BY REFERENCE: the authoritative torch modeling+ # hard-codes the direct q_proj without bias (use_qkv_bias+ # applies only to the LoRA a-projections and kv_a/dense) —+ # mirroring it exactly, not an omission.+ self.q_proj = nn.Linear(+ hidden, self.num_heads * self.qk_head_dim, bias=False+ )+ else:+ self.q_a_proj = nn.Linear(hidden, self.q_lora_rank, bias=bias)+ self.q_a_layernorm = nn.RMSNorm(self.q_lora_rank, eps=args.rms_norm_eps)+ self.q_b_proj = nn.Linear(+ self.q_lora_rank, self.num_heads * self.qk_head_dim, bias=False+ )+ self.kv_a_proj_with_mqa = nn.Linear(+ hidden, self.kv_lora_rank + self.qk_rope_head_dim, bias=bias+ )+ self.kv_a_layernorm = nn.RMSNorm(self.kv_lora_rank, eps=args.rms_norm_eps)+ self.kv_b_proj = nn.Linear(+ self.kv_lora_rank,+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),+ bias=False,+ )+ if self.gate_kind == "head_wise":+ self.g_proj = nn.Linear(hidden, self.num_heads, bias=False)+ elif self.gate_kind == "element_wise":+ self.g_proj = nn.Linear(+ hidden, self.num_heads * self.v_head_dim, bias=False+ )+ # Checkpoint names the output projection ``dense``.+ self.dense = nn.Linear(self.num_heads * self.v_head_dim, hidden, bias=bias)+ # RoPE follows the upstream torch implementation exactly:+ # transformers 4.x ``default`` (rope_scaling=null) computes+ # inv_freq = 1/theta^(2i/dim), emb = cat((freqs, freqs)), then+ # apply_rotary_pos_emb_interleave rotates consecutive pairs+ # (x[2i], x[2i+1]) by pos*inv_freq[i]. We hand-roll it+ # (_rope_interleave_torch) because mx.fast.rope's offset is a single+ # absolute position: prefill with offset=0 would skip rotation.+ if args.rope_scaling:+ raise NotImplementedError(+ "bailing_hybrid: rope_scaling is not supported by the "+ f"vendored backbone yet (got {args.rope_scaling!r})"+ )+ self.rope_theta = args.rope_theta++ def __call__(+ self,+ x: mx.array,+ mask: mx.array | None = None,+ cache: Any | None = None,+ ) -> mx.array:+ B, L, _ = x.shape++ if self.q_lora_rank is None:+ q = self.q_proj(x)+ else:+ q = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(x)))+ q = q.reshape(B, L, self.num_heads, self.qk_head_dim).transpose(0, 2, 1, 3)+ q_nope, q_pe = mx.split(q, [self.qk_nope_head_dim], axis=-1)++ compressed = self.kv_a_proj_with_mqa(x)+ kv_latent, k_pe = mx.split(compressed, [self.kv_lora_rank], axis=-1)+ kv_latent = self.kv_a_layernorm(kv_latent)+ k_pe = k_pe.reshape(B, L, 1, self.qk_rope_head_dim).transpose(0, 2, 1, 3)++ kv = self.kv_b_proj(kv_latent)+ kv = kv.reshape(+ B, L, self.num_heads, self.qk_nope_head_dim + self.v_head_dim+ ).transpose(0, 2, 1, 3)+ k_nope, values = mx.split(kv, [self.qk_nope_head_dim], axis=-1)++ offset = cache.offset if cache is not None else 0+ positions = mx.arange(L) + offset+ q_pe = _rope_interleave_torch(q_pe, positions, self.rope_theta)+ k_pe = _rope_interleave_torch(k_pe, positions, self.rope_theta)+ k_pe = mx.broadcast_to(k_pe, (B, self.num_heads, L, self.qk_rope_head_dim))++ queries = mx.concatenate([q_nope, q_pe], axis=-1)+ keys = mx.concatenate([k_nope, k_pe], axis=-1)++ if cache is not None:+ keys, values = cache.update_and_fetch(keys, values)++ out = scaled_dot_product_attention(+ queries,+ values=values,+ keys=keys,+ cache=cache,+ scale=self.scale,+ mask=mask,+ )+ out = out.transpose(0, 2, 1, 3).reshape(B, L, -1)+ if self.gate_kind == "head_wise":+ gate = mx.sigmoid(self.g_proj(x)) # [B, L, H]+ out = out.reshape(B, L, self.num_heads, self.v_head_dim)+ out = out * gate[..., None]+ out = out.reshape(B, L, -1)+ elif self.gate_kind == "element_wise":+ out = out * mx.sigmoid(self.g_proj(x))+ return self.dense(out)+++class BailingMLP(nn.Module):+ def __init__(self, args: ModelArgs, intermediate: int):+ super().__init__()+ self.gate_proj = nn.Linear(args.hidden_size, intermediate, bias=False)+ self.up_proj = nn.Linear(args.hidden_size, intermediate, bias=False)+ self.down_proj = nn.Linear(intermediate, args.hidden_size, bias=False)++ def __call__(self, x: mx.array) -> mx.array:+ return self.down_proj(nn.silu(self.gate_proj(x)) * self.up_proj(x))+++class BailingGate(nn.Module):+ """Sigmoid-score router with noaux_tc expert-bias group selection.++ DeepSeek-V3 routing law (torch ref ``BailingMoeV3Gate``): selection+ scores = sigmoid(logits) + expert_bias, grouped into ``n_group``;+ the ``topk_group`` best groups (by sum of each group's top-2+ selection scores) survive; the top-k experts within the surviving+ groups are chosen by selection score; the WEIGHTS are the raw+ sigmoid scores of the chosen experts (bias excluded), normalized if+ ``norm_topk_prob`` and scaled by ``routed_scaling_factor``.+ """++ def __init__(self, args: ModelArgs):+ super().__init__()+ self.args = args+ self.weight = mx.zeros((args.num_experts, args.hidden_size))+ if args.moe_router_enable_expert_bias:+ self.expert_bias = mx.zeros((args.num_experts,))++ def __call__(self, x: mx.array) -> tuple[mx.array, mx.array]:+ a = self.args+ scores = mx.sigmoid((x @ self.weight.T).astype(mx.float32))+ select = scores+ if a.moe_router_enable_expert_bias:+ select = scores + self.expert_bias++ B = select.shape[:-1]+ k_drop = a.n_group - a.topk_group+ if k_drop > 0:+ grouped = select.reshape(*B, a.n_group, a.num_experts // a.n_group)+ # Group score = sum of the top-2 selection scores in the group.+ top2 = mx.topk(grouped, 2, axis=-1)+ group_scores = top2.sum(axis=-1)+ drop = mx.argpartition(group_scores, kth=k_drop - 1, axis=-1)[..., :k_drop]+ # ``put_along_axis`` broadcasts the trailing size-1 index dim+ # across the experts-per-group axis (numpy semantics), so the+ # single write masks EVERY expert slot of each dropped group —+ # unlike torch scatter, no expand is needed. Pinned by+ # ``test_gate_group_drop_masks_whole_group`` and by the+ # reference parity run (1.5e-6 on a drop-path config).+ masked = mx.put_along_axis(+ grouped,+ mx.expand_dims(drop, -1),+ mx.array(-float("inf"), grouped.dtype),+ axis=-2,+ )+ select = masked.reshape(*B, a.num_experts)+ # topk_group == n_group keeps every group — nothing to drop+ # (codex r3 #1: argpartition(kth=-1) would fault).++ k = a.num_experts_per_tok+ idx = mx.argpartition(-select, kth=k - 1, axis=-1)[..., :k]+ w = mx.take_along_axis(scores, idx, axis=-1)+ if a.norm_topk_prob:+ w = w / (w.sum(axis=-1, keepdims=True) + 1e-20)+ w = w * a.routed_scaling_factor+ return idx, w+++class BailingPrerouter(nn.Module):+ """One-MoE-block-ahead expert selector (arXiv 2308.12066 style).++ Input is the current layer's MoE input (post-attention RMSNorm output)+ plus optional current-layer top-8 one-hot and prev-token top-8 one-hot+ (v4/v5 feature set). Output is 128 logits. Selection reuses the+ sigmoid + group-limited top-k semantics of ``BailingGate`` (no expert+ bias), so at deployment the pre-gate can fully replace the router for+ its target layer.+ """++ def __init__(+ self,+ args: ModelArgs,+ hidden: int | None = None,+ input_dim: int | None = None,+ ):+ super().__init__()+ self.args = args+ self.hidden = hidden or args.prerouter_hidden+ self.input_dim = input_dim or args.hidden_size+ self.fc1 = nn.Linear(self.input_dim, self.hidden, bias=False)+ self.fc2 = nn.Linear(self.hidden, args.num_experts, bias=False)+ # v4/v5 residual linear path (zero by default; trained in v5).+ self.linear_init = nn.Linear(+ self.input_dim, args.num_experts, bias=False+ )+ self.linear_init.weight = mx.zeros(+ (args.num_experts, self.input_dim), dtype=mx.float32+ )++ def __call__(+ self,+ x: mx.array,+ topk_oh: mx.array | None = None,+ prev_oh: mx.array | None = None,+ ) -> mx.array:+ dtype = self.fc1.weight.dtype+ feats = [x.astype(dtype)]+ if topk_oh is not None:+ feats.append(topk_oh.astype(dtype))+ if prev_oh is not None:+ feats.append(prev_oh.astype(dtype))+ xi = mx.concatenate(feats, axis=-1)+ h = nn.gelu(self.fc1(xi))+ return self.fc2(h) + self.linear_init(xi)+++class BailingSparseMoE(nn.Module):+ def __init__(+ self,+ args: ModelArgs,+ prerouter: BailingPrerouter | None = None,+ use_prerouter: bool = False,+ ):+ super().__init__()+ self.args = args+ self.gate = BailingGate(args)+ self.prerouter = prerouter+ self.use_prerouter = use_prerouter+ self.last_topk: mx.array | None = None+ self.prev_topk_oh: mx.array | None = None+ # v5 was trained with the TEACHER (original router) top-8 as the+ # pre-gate's "current layer top-8" feature, so at decode we compute+ # the original gate top-8 for the feature even when the pre-gate+ # selects the executed experts. Set to "executed" to use the+ # pre-gate's own selection instead (post-SFT ce_only semantics).+ self.feature_topk = os.environ.get(+ "PREROUTER_FEATURE_TOPK", "teacher")+ # Same-token intra path computes the teacher feature inside the+ # after-attention callback; the per-layer gate re-computation here+ # would be dead work (nothing consumes last_topk in intra mode).+ self.intra_mode = os.environ.get("PREROUTER_INTRA", "0") == "1"+ self.experts = SwitchGLU(+ args.hidden_size, args.moe_intermediate_size, args.num_experts+ )+ self.shared_experts = BailingMLP(+ args,+ args.moe_shared_expert_intermediate_size * args.num_shared_experts,+ )++ def _select_from_logits(self, logits: mx.array) -> tuple[mx.array, mx.array]:+ """Same routing math as BailingGate, but from pre-gate logits+ (expert bias excluded)."""+ a = self.args+ scores = mx.sigmoid(logits.astype(mx.float32))+ B = scores.shape[:-1]+ k_drop = a.n_group - a.topk_group+ if k_drop > 0:+ grouped = scores.reshape(+ *B, a.n_group, a.num_experts // a.n_group+ )+ top2 = mx.topk(grouped, 2, axis=-1)+ group_scores = top2.sum(axis=-1)+ drop = mx.argpartition(+ group_scores, kth=k_drop - 1, axis=-1+ )[..., :k_drop]+ masked = mx.put_along_axis(+ grouped,+ mx.expand_dims(drop, -1),+ mx.array(-float("inf"), grouped.dtype),+ axis=-2,+ )+ select = masked.reshape(*B, a.num_experts)+ else:+ select = scores+ k = a.num_experts_per_tok+ idx = mx.argpartition(-select, kth=k - 1, axis=-1)[..., :k]+ w = mx.take_along_axis(scores, idx, axis=-1)+ if a.norm_topk_prob:+ w = w / (w.sum(axis=-1, keepdims=True) + 1e-20)+ w = w * a.routed_scaling_factor+ return idx, w++ def __call__(+ self, x: mx.array, prev_prerouter_logits: mx.array | None = None+ ) -> mx.array:+ if self.use_prerouter and prev_prerouter_logits is not None:+ idx, w = self._select_from_logits(prev_prerouter_logits)+ if (not self.intra_mode+ and self.feature_topk == "teacher"):+ feat_idx, _ = self.gate(x)+ else:+ feat_idx = idx+ else:+ idx, w = self.gate(x)+ feat_idx = idx+ self.last_topk = feat_idx if not self.intra_mode else None+ routed = self.experts(x, idx)+ out = (routed * w[..., None].astype(routed.dtype)).sum(axis=-2)+ return out + self.shared_experts(x)+++class BailingDecoderLayer(nn.Module):+ def __init__(self, args: ModelArgs, layer_idx: int):+ super().__init__()+ self.layer_idx = layer_idx+ self.is_mla = args.is_mla_layer(layer_idx)+ if self.is_mla:+ self.attention = BailingMLA(args)+ else:+ self.attention = BailingKDA(args, layer_idx)+ self.prerouter_enabled = bool(getattr(args, "prerouter_enabled", False))+ self.prerouter_start_layer = int(getattr(args, "prerouter_start_layer", 7))+ self.prerouter_logits: mx.array | None = None+ self.m_in_cache: mx.array | None = None+ self.after_attention_cb = None+ self.has_prerouter = (+ self.prerouter_enabled+ and layer_idx >= self.prerouter_start_layer - 1+ and layer_idx < args.num_hidden_layers - 1+ )+ self.use_prerouter = (+ self.prerouter_enabled+ and layer_idx >= self.prerouter_start_layer+ and layer_idx < args.num_hidden_layers+ )+ if layer_idx >= args.first_k_dense_replace:+ self.mlp = BailingSparseMoE(+ args,+ prerouter=(+ BailingPrerouter(args)+ if self.has_prerouter+ else None+ ),+ use_prerouter=self.use_prerouter,+ )+ else:+ self.mlp = BailingMLP(args, args.intermediate_size)+ self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)+ self.post_attention_layernorm = nn.RMSNorm(+ args.hidden_size, eps=args.rms_norm_eps+ )++ def __call__(+ self,+ x: mx.array,+ mask: mx.array | None = None,+ cache: Any | None = None,+ prev_prerouter_logits: mx.array | None = None,+ ) -> mx.array:+ h = x + self.attention(self.input_layernorm(x), mask, cache)+ m_in = self.post_attention_layernorm(h)+ if self.has_prerouter:+ # Defer the pre-gate MLP to a single post-eval batch (PrerouterStager+ # stacks every layer's m_in), keeping the forward graph lean.+ self.m_in_cache = m_in+ self.prerouter_logits = None+ else:+ self.prerouter_logits = None+ self.m_in_cache = None+ if self.after_attention_cb is not None:+ self.after_attention_cb(self.layer_idx, m_in)+ if isinstance(self.mlp, BailingSparseMoE):+ out = self.mlp(m_in, prev_prerouter_logits=prev_prerouter_logits)+ else:+ out = self.mlp(m_in)+ return h + out+++class BailingModel(nn.Module):+ def __init__(self, args: ModelArgs):+ super().__init__()+ self.args = args+ self.word_embeddings = nn.Embedding(args.vocab_size, args.hidden_size)+ self.layers = [+ BailingDecoderLayer(args, i) for i in range(args.num_hidden_layers)+ ]+ self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)+ self.first_mla_idx = next(+ i for i in range(args.num_hidden_layers) if args.is_mla_layer(i)+ )++ def __call__(+ self,+ inputs: mx.array,+ cache=None,+ input_embeddings: mx.array | None = None,+ after_layer_cb=None,+ before_layer_cb=None,+ async_eval_per_layer: bool = False,+ prerouter_cache: dict[int, mx.array] | None = None,+ ) -> mx.array:+ h = (+ input_embeddings+ if input_embeddings is not None+ else self.word_embeddings(inputs)+ )+ if cache is None:+ cache = [None] * len(self.layers)+ mla_mask = create_attention_mask(h, cache[self.first_mla_idx])+ # Hidden-state clip (LING_HIDDEN_CLIP, e.g. 1000): clamps each+ # layer's output to [-v, +v] and zeroes NaN entries so a single+ # fp16 overflow inside one layer cannot poison the whole network+ # (observed: decode blowup -> all-NaN logits -> argmax falls back+ # to token 0 ('!') death spiral). Off by default.+ # (v7-deploy NAN_BANG_COLLAPSE_FIX.md parity.)+ _clip_v = float(os.environ.get("LING_HIDDEN_CLIP", "0"))+ for li, (layer, c) in enumerate(zip(self.layers, cache)):+ if before_layer_cb is not None:+ before_layer_cb(li)+ mask = mla_mask if layer.is_mla else None+ prev_prerouter_logits = None+ if (+ getattr(layer, "use_prerouter", False)+ and prerouter_cache is not None+ and li in prerouter_cache+ ):+ prev_prerouter_logits = prerouter_cache[li]+ h = layer(h, mask, c, prev_prerouter_logits)+ if _clip_v > 0:+ h = mx.where(+ mx.isnan(h), mx.zeros_like(h),+ mx.clip(h, -_clip_v, _clip_v))+ if after_layer_cb is not None:+ after_layer_cb(li, h)+ if async_eval_per_layer:+ # Fire this layer's graph on the GPU while the CPU keeps+ # building the next layer (overlaps the whole-layer weight+ # load of the E3b prefill path). The final logits eval+ # still waits for everything.+ mx.async_eval(h)+ return self.norm(h)+++class Model(nn.Module):+ def __init__(self, args: ModelArgs):+ super().__init__()+ self.args = args+ self.model_type = args.model_type+ self.model = BailingModel(args)+ self.tie_word_embeddings = args.tie_word_embeddings+ if not self.tie_word_embeddings:+ self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False)++ def __call__(+ self,+ inputs: mx.array,+ cache=None,+ input_embeddings: mx.array | None = None,+ after_layer_cb=None,+ ) -> mx.array:+ h = self.model(inputs, cache, input_embeddings, after_layer_cb)+ if self.tie_word_embeddings:+ return self.model.word_embeddings.as_linear(h)+ return self.lm_head(h)++ def sanitize(self, weights):+ """Stack per-expert weights into SwitchGLU layout and drop any+ MTP head (flash/2.6 ship ``num_nextn_predict_layers`` layers we+ do not serve)."""+ weights = {+ k: v for k, v in weights.items() if ".mtp_" not in k and "mtp." not in k+ }+ n_layers = self.args.num_hidden_layers+ for li in range(n_layers):+ prefix = f"model.layers.{li}"+ if f"{prefix}.mlp.experts.0.gate_proj.weight" not in weights and (+ f"{prefix}.mlp.experts.0.gate_proj.scales" not in weights+ ):+ continue+ for m in ("gate_proj", "up_proj", "down_proj"):+ for part in ("weight", "scales", "biases"):+ key0 = f"{prefix}.mlp.experts.0.{m}.{part}"+ if key0 not in weights:+ continue+ joined = mx.stack(+ [+ weights.pop(f"{prefix}.mlp.experts.{e}.{m}.{part}")+ for e in range(self.args.num_experts)+ ]+ )+ weights[f"{prefix}.mlp.experts.{m}.{part}"] = joined+ # Conv weights: the checkpoint stores them at ``*_conv1d.weight``+ # in torch depthwise layout [C, 1, ksize] (or flat [C, ksize]);+ # our ShortConv1d nests an nn.Conv1d at ``.conv`` expecting+ # [C, ksize, 1].+ for k in list(weights):+ if k.endswith("_conv1d.weight"):+ w = weights.pop(k)+ if w.ndim == 2:+ w = w[..., None]+ elif w.ndim == 3 and w.shape[1] == 1:+ w = w.swapaxes(1, 2)+ weights[k[: -len(".weight")] + ".conv.weight"] = w+ return weights++ @property+ def layers(self):+ return self.model.layers++ def make_cache(self):+ return [+ KVCache() if layer.is_mla else ArraysCache(size=4)+ for layer in self.model.layers+ ]++ @property+ def quant_predicate(self):+ def predicate(path, module):+ # Short-conv weights stay fp — tiny tensors, and quantized+ # depthwise conv has no parity coverage (codex r3 #2).+ if "_conv1d" in path:+ return False+ # Router quality is precision-critical (deepseek-v3+ # precedent): keep the gate at 8-bit.+ if "mlp.gate" in path and "gate_proj" not in path:+ return {"group_size": 64, "bits": 8}+ return True++ return predicatediff --git a/src/edge0/backends/mlx/_impl/qwen3_5.py b/src/edge0/backends/mlx/_impl/qwen3_5.pynew file mode 100644index 0000000..c5cf965--- /dev/null+++ b/src/edge0/backends/mlx/_impl/qwen3_5.py@@ -0,0 +1,545 @@+# Copyright © 2026 Apple Inc.+#+# edge0 vendored copy of mlx-lm 0.31.0 ``mlx_lm/models/qwen3_5.py``,+# byte-identical EXCEPT: (1) the import block rewritten to absolute+# paths — infra bits (base/cache/gated_delta) point at the pinned+# mlx_lm package, sibling model files point at this vendored package so+# the prerouter block patch lands on OUR ``Qwen3NextSparseMoeBlock``;+# (2) ``Qwen3_5TextModel.__call__`` gained the edge0 engine callback+# hooks (before/after_layer_cb + async_eval_per_layer).++from dataclasses import dataclass, field+from typing import Any, Dict, List, Optional, Union++import mlx.core as mx+import mlx.nn as nn+from mlx.nn.layers.distributed import shard_inplace, shard_linear, sum_gradients+from mlx.utils import tree_map++from mlx_lm.models.base import (+ BaseModelArgs,+ create_attention_mask,+ create_ssm_mask,+)+from mlx_lm.models.cache import ArraysCache, KVCache+from mlx_lm.models.gated_delta import gated_delta_update+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextAttention as Attention+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextMLP as MLP+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextRMSNormGated as RMSNormGated+from edge0.backends.mlx._impl.qwen3_next import Qwen3NextSparseMoeBlock as SparseMoeBlock+++@dataclass+class TextModelArgs(BaseModelArgs):+ model_type: str = ""+ hidden_size: int = 4096+ intermediate_size: int = 14336+ num_hidden_layers: int = 32+ num_attention_heads: int = 32+ rms_norm_eps: float = 1e-6+ vocab_size: int = 151936+ num_key_value_heads: int = 8+ max_position_embeddings: int = 131072+ linear_num_value_heads: int = 64+ linear_num_key_heads: int = 16+ linear_key_head_dim: int = 192+ linear_value_head_dim: int = 128+ linear_conv_kernel_dim: int = 4+ tie_word_embeddings: bool = False+ attention_bias: bool = False+ head_dim: Optional[int] = None+ full_attention_interval: int = 4++ # MoE fields (optional, for Qwen3_5MoeForConditionalGeneration)+ num_experts: int = 0+ num_experts_per_tok: int = 0+ decoder_sparse_step: int = 1+ shared_expert_intermediate_size: int = 0+ moe_intermediate_size: int = 0+ norm_topk_prob: bool = True++ # Rope parameters+ rope_parameters: Optional[Dict[str, Union[float, str, bool, List[int]]]] = field(+ default_factory=lambda: {+ "type": "default",+ "mrope_section": [11, 11, 10],+ "rope_theta": 100000,+ "partial_rotary_factor": 0.25,+ }+ )++ # Derived from rope_parameters (set in __post_init__)+ partial_rotary_factor: float = 0.25+ rope_theta: float = 100000.0+ rope_scaling: Optional[Dict[str, Union[float, str]]] = None++ def __post_init__(self):+ if self.head_dim is None:+ self.head_dim = self.hidden_size // self.num_attention_heads++ if self.rope_parameters:+ if (+ "type" not in self.rope_parameters+ and "rope_type" in self.rope_parameters+ ):+ self.rope_parameters["type"] = self.rope_parameters.pop("rope_type")++ self.partial_rotary_factor = self.rope_parameters.get(+ "partial_rotary_factor", 0.25+ )+ self.rope_theta = self.rope_parameters.get("rope_theta", 100000.0)+ self.rope_scaling = self.rope_parameters+++class GatedDeltaNet(nn.Module):+ def __init__(self, config: TextModelArgs):+ super().__init__()+ self.hidden_size = config.hidden_size+ self.num_v_heads = config.linear_num_value_heads+ self.num_k_heads = config.linear_num_key_heads+ self.head_k_dim = config.linear_key_head_dim+ self.head_v_dim = config.linear_value_head_dim+ self.key_dim = self.head_k_dim * self.num_k_heads+ self.value_dim = self.head_v_dim * self.num_v_heads+ if self.num_v_heads % self.num_k_heads != 0:+ raise ValueError(+ f"num_v_heads ({self.num_v_heads}) must be divisible by num_k_heads ({self.num_k_heads})"+ )++ self.conv_kernel_size = config.linear_conv_kernel_dim+ self.layer_norm_epsilon = config.rms_norm_eps++ self.conv_dim = self.key_dim * 2 + self.value_dim+ self.conv1d = nn.Conv1d(+ in_channels=self.conv_dim,+ out_channels=self.conv_dim,+ bias=False,+ kernel_size=self.conv_kernel_size,+ groups=self.conv_dim,+ padding=0,+ )++ self.in_proj_qkv = nn.Linear(+ self.hidden_size, self.key_dim * 2 + self.value_dim, bias=False+ )+ self.in_proj_z = nn.Linear(self.hidden_size, self.value_dim, bias=False)+ self.in_proj_b = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)+ self.in_proj_a = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)++ self.dt_bias = mx.ones(self.num_v_heads)++ A = mx.random.uniform(low=0, high=16, shape=(self.num_v_heads,))+ self.A_log = mx.log(A)++ self.norm = RMSNormGated(self.head_v_dim, eps=self.layer_norm_epsilon)++ self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)++ self.sharding_group = None++ def __call__(+ self,+ inputs: mx.array,+ mask: Optional[mx.array] = None,+ cache: Optional[Any] = None,+ ) -> mx.array:+ B, S, _ = inputs.shape++ if self.sharding_group is not None:+ inputs = sum_gradients(self.sharding_group)(inputs)++ qkv = self.in_proj_qkv(inputs)+ z = self.in_proj_z(inputs).reshape(B, S, self.num_v_heads, self.head_v_dim)+ b = self.in_proj_b(inputs)+ a = self.in_proj_a(inputs)++ if cache is not None and cache[0] is not None:+ conv_state = cache[0]+ else:+ conv_state = mx.zeros(+ (B, self.conv_kernel_size - 1, self.conv_dim),+ dtype=inputs.dtype,+ )++ if mask is not None:+ qkv = mx.where(mask[..., None], qkv, 0)+ conv_input = mx.concatenate([conv_state, qkv], axis=1)+ if cache is not None:+ cache[0] = conv_input[:, -(self.conv_kernel_size - 1) :]+ conv_out = nn.silu(self.conv1d(conv_input))++ q, k, v = [+ t.reshape(B, S, h, d)+ for t, h, d in zip(+ mx.split(conv_out, [self.key_dim, 2 * self.key_dim], -1),+ [self.num_k_heads, self.num_k_heads, self.num_v_heads],+ [self.head_k_dim, self.head_k_dim, self.head_v_dim],+ )+ ]++ state = cache[1] if cache else None+ inv_scale = k.shape[-1] ** -0.5+ q = (inv_scale**2) * mx.fast.rms_norm(q, None, 1e-6)+ k = inv_scale * mx.fast.rms_norm(k, None, 1e-6)++ out, state = gated_delta_update(+ q,+ k,+ v,+ a,+ b,+ self.A_log,+ self.dt_bias,+ state,+ mask,+ use_kernel=not self.training,+ )++ if cache is not None:+ cache[1] = state++ out = self.norm(out, z)+ out = self.out_proj(out.reshape(B, S, -1))++ if self.sharding_group is not None:+ out = mx.distributed.all_sum(out, group=self.sharding_group)++ return out+++class DecoderLayer(nn.Module):+ def __init__(self, args: TextModelArgs, layer_idx: int):+ super().__init__()+ self.is_linear = (layer_idx + 1) % args.full_attention_interval != 0+ if self.is_linear:+ self.linear_attn = GatedDeltaNet(args)+ else:+ self.self_attn = Attention(args)++ self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)+ self.post_attention_layernorm = nn.RMSNorm(+ args.hidden_size, eps=args.rms_norm_eps+ )++ if args.num_experts > 0:+ self.mlp = SparseMoeBlock(args)+ else:+ self.mlp = MLP(args.hidden_size, args.intermediate_size)++ def __call__(+ self,+ x: mx.array,+ mask: Optional[mx.array] = None,+ cache: Optional[Any] = None,+ ) -> mx.array:+ if self.is_linear:+ r = self.linear_attn(self.input_layernorm(x), mask, cache)+ else:+ r = self.self_attn(self.input_layernorm(x), mask, cache)+ h = x + r+ out = h + self.mlp(self.post_attention_layernorm(h))+ return out+++class Qwen3_5TextModel(nn.Module):+ def __init__(self, args: TextModelArgs):+ super().__init__()+ self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)+ self.layers = [+ DecoderLayer(args=args, layer_idx=i) for i in range(args.num_hidden_layers)+ ]+ self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)+ self.ssm_idx = 0+ self.fa_idx = args.full_attention_interval - 1++ def __call__(+ self,+ inputs: mx.array,+ cache: Optional[Any] = None,+ input_embeddings: Optional[mx.array] = None,+ before_layer_cb=None,+ after_layer_cb=None,+ async_eval_per_layer: bool = False,+ ) -> mx.array:+ if input_embeddings is not None:+ hidden_states = input_embeddings+ else:+ hidden_states = self.embed_tokens(inputs)++ if cache is None:+ cache = [None] * len(self.layers)++ fa_mask = create_attention_mask(hidden_states, cache[self.fa_idx])+ ssm_mask = create_ssm_mask(hidden_states, cache[self.ssm_idx])++ for li, (layer, c) in enumerate(zip(self.layers, cache)):+ if before_layer_cb is not None:+ before_layer_cb(li)+ mask = ssm_mask if layer.is_linear else fa_mask+ hidden_states = layer(hidden_states, mask=mask, cache=c)+ if after_layer_cb is not None:+ after_layer_cb(li, hidden_states)+ if async_eval_per_layer:+ # Fire this layer's graph on the GPU while the CPU keeps+ # building the next layer (overlaps the whole-layer weight+ # load of the E3b prefill path). The final logits eval+ # still waits for everything.+ mx.async_eval(hidden_states)++ return self.norm(hidden_states)+++class TextModel(nn.Module):+ def __init__(self, args: TextModelArgs):+ super().__init__()+ self.args = args+ self.model_type = args.model_type+ self.model = Qwen3_5TextModel(args)+ if not args.tie_word_embeddings:+ self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False)++ def __call__(+ self,+ inputs: mx.array,+ cache: Optional[Any] = None,+ input_embeddings: Optional[mx.array] = None,+ ) -> mx.array:+ out = self.model(inputs, cache, input_embeddings=input_embeddings)+ if self.args.tie_word_embeddings:+ out = self.model.embed_tokens.as_linear(out)+ else:+ out = self.lm_head(out)+ return out++ @property+ def layers(self):+ return self.model.layers++ def make_cache(self):+ return [ArraysCache(size=2) if l.is_linear else KVCache() for l in self.layers]++ def sanitize(self, weights):+ has_mtp_weights = any("mtp." in k for k in weights)+ has_unsanitized_conv1d = any(+ "conv1d.weight" in k and v.shape[-1] != 1 for k, v in weights.items()+ )+ should_shift_norm_weights = has_mtp_weights or has_unsanitized_conv1d+ weights = {k: v for k, v in weights.items() if "mtp." not in k}++ if self.args.tie_word_embeddings:+ weights.pop("lm_head.weight", None)++ norm_keys = (+ ".input_layernorm.weight",+ ".post_attention_layernorm.weight",+ "model.norm.weight",+ ".q_norm.weight",+ ".k_norm.weight",+ )+ for k, v in weights.items():+ if "conv1d.weight" in k and v.shape[-1] != 1:+ weights[k] = v.moveaxis(2, 1)+ if should_shift_norm_weights and any(k.endswith(sfx) for sfx in norm_keys):+ if v.ndim == 1:+ weights[k] = v + 1.0+ return weights++ @property+ def quant_predicate(self):+ if self.args.num_experts <= 0:+ return None++ def predicate(path, _):+ if path.endswith("mlp.gate") or path.endswith("shared_expert_gate"):+ return {"group_size": 64, "bits": 8}+ return True++ return predicate++ @property+ def cast_predicate(self):+ def predicate(path: str):+ if path.endswith("A_log"):+ return False+ return True++ return predicate+++@dataclass+class ModelArgs(BaseModelArgs):+ model_type: str+ text_config: dict++ @classmethod+ def from_dict(cls, params):+ if "text_config" not in params:+ return cls(model_type=params["model_type"], text_config=params)+ return super().from_dict(params)+++class Model(nn.Module):+ def __init__(self, args: ModelArgs):+ super().__init__()+ self.args = args+ self.model_type = args.model_type+ self.language_model = TextModel(TextModelArgs.from_dict(args.text_config))++ def __call__(+ self,+ inputs: mx.array,+ cache=None,+ input_embeddings: Optional[mx.array] = None,+ ):+ return self.language_model(+ inputs, cache=cache, input_embeddings=input_embeddings+ )++ def sanitize(self, weights):+ sanitized = {}+ for key, value in weights.items():+ if key.startswith("vision_tower") or key.startswith("model.visual"):+ continue+ if key.startswith("model.visual"):+ continue+ if key.startswith("model.language_model"):+ key = key.replace("model.language_model", "language_model.model")+ elif key.startswith("language_model."):+ pass+ else:+ key = "language_model." + key+ sanitized[key] = value+ return self.language_model.sanitize(sanitized)++ def shard(self, group=None):+ group = group or mx.distributed.init()+ N = group.size()+ rank = group.rank()++ # A sharding factory for the convolution in gated delta net+ def conv_sharding(key_dim):+ return lambda p, w: (0, [key_dim, 2 * key_dim])++ def repeat_kv_layer_inplace(layer, h):+ # No repeat needed cause we have more heads than nodes+ if N <= h:+ return++ # Repeat function to apply to the layer weights+ def _repeat(p):+ s = p.shape+ p = p.reshape(h, s[0] // h, *s[1:])+ p = mx.repeat(p, N // h, axis=0)+ p = p.reshape(-1, *s[1:])+ return p++ layer.update(tree_map(_repeat, layer.parameters()))++ for layer in self.layers:+ # Linear attention+ if layer.is_linear:+ kd = layer.linear_attn.key_dim+ layer.linear_attn.sharding_group = group+ shard_inplace(layer.linear_attn.conv1d, conv_sharding(kd), group=group)+ layer.linear_attn.conv1d.groups //= N+ shard_inplace(+ layer.linear_attn.in_proj_qkv,+ "all-to-sharded",+ segments=[kd, 2 * kd],+ group=group,+ )+ shard_inplace(+ layer.linear_attn.in_proj_z, "all-to-sharded", group=group+ )+ shard_inplace(+ layer.linear_attn.in_proj_b, "all-to-sharded", group=group+ )+ shard_inplace(+ layer.linear_attn.in_proj_a, "all-to-sharded", group=group+ )+ layer.linear_attn.dt_bias = mx.contiguous(+ mx.split(layer.linear_attn.dt_bias, N)[rank]+ )+ layer.linear_attn.A_log = mx.contiguous(+ mx.split(layer.linear_attn.A_log, N)[rank]+ )+ shard_inplace(layer.linear_attn.out_proj, "sharded-to-all", group=group)+ layer.linear_attn.num_k_heads //= N+ layer.linear_attn.num_v_heads //= N+ layer.linear_attn.key_dim //= N+ layer.linear_attn.value_dim //= N+ layer.linear_attn.conv_dim //= N++ # Softmax attention+ else:+ layer.self_attn.o_proj = shard_linear(+ layer.self_attn.o_proj, "sharded-to-all", group=group+ )+ layer.self_attn.q_proj = shard_linear(+ layer.self_attn.q_proj, "all-to-sharded", group=group+ )+ repeat_kv_layer_inplace(+ layer.self_attn.k_proj, layer.self_attn.num_key_value_heads+ )+ repeat_kv_layer_inplace(+ layer.self_attn.v_proj, layer.self_attn.num_key_value_heads+ )+ layer.self_attn.k_proj = shard_linear(+ layer.self_attn.k_proj, "all-to-sharded", group=group+ )+ layer.self_attn.v_proj = shard_linear(+ layer.self_attn.v_proj, "all-to-sharded", group=group+ )+ layer.self_attn.num_attention_heads //= N+ layer.self_attn.num_key_value_heads = max(+ 1, layer.self_attn.num_key_value_heads // N+ )++ # MLP+ if isinstance(layer.mlp, MLP):+ layer.mlp.gate_proj = shard_linear(+ layer.mlp.gate_proj, "all-to-sharded", group=group+ )+ layer.mlp.down_proj = shard_linear(+ layer.mlp.down_proj, "sharded-to-all", group=group+ )+ layer.mlp.up_proj = shard_linear(+ layer.mlp.up_proj, "all-to-sharded", group=group+ )++ # MoE+ else:+ layer.mlp.sharding_group = group+ shard_inplace(+ layer.mlp.shared_expert.gate_proj, "all-to-sharded", group=group+ )+ shard_inplace(+ layer.mlp.shared_expert.down_proj, "sharded-to-all", group=group+ )+ shard_inplace(+ layer.mlp.shared_expert.up_proj, "all-to-sharded", group=group+ )+ shard_inplace(+ layer.mlp.switch_mlp.gate_proj, "all-to-sharded", group=group+ )+ shard_inplace(+ layer.mlp.switch_mlp.down_proj, "sharded-to-all", group=group+ )+ shard_inplace(+ layer.mlp.switch_mlp.up_proj, "all-to-sharded", group=group+ )++ @property+ def layers(self):+ return self.language_model.model.layers++ def make_cache(self):+ return self.language_model.make_cache()++ @property+ def quant_predicate(self):+ return self.language_model.quant_predicate++ @property+ def cast_predicate(self):+ return self.language_model.cast_predicatediff --git a/src/edge0/backends/mlx/_impl/qwen3_5_moe.py b/src/edge0/backends/mlx/_impl/qwen3_5_moe.pynew file mode 100644index 0000000..7930733--- /dev/null+++ b/src/edge0/backends/mlx/_impl/qwen3_5_moe.py@@ -0,0 +1,56 @@+# Copyright © 2026 Apple Inc.+#+# edge0 vendored copy of mlx-lm 0.31.0 ``mlx_lm/models/qwen3_5_moe.py``,+# byte-identical EXCEPT the import block (absolute paths; the qwen3_5+# sibling resolves to this vendored package).++from dataclasses import dataclass++from mlx_lm.models.base import BaseModelArgs+from edge0.backends.mlx._impl.qwen3_5 import Model as Qwen3_5Model+++@dataclass+class ModelArgs(BaseModelArgs):+ model_type: str+ text_config: dict++ @classmethod+ def from_dict(cls, params):+ if "text_config" not in params:+ return cls(model_type=params["model_type"], text_config=params)+ return super().from_dict(params)+++class Model(Qwen3_5Model):++ def sanitize(self, weights):+ new_weights = {}+ for key, value in weights.items():+ if key.startswith("vision_tower") or key.startswith("model.visual"):+ continue+ if key.startswith("model.language_model"):+ key = key.replace("model.language_model", "language_model.model")+ elif key.startswith("language_model."):+ pass+ else:+ key = "language_model." + key+ new_weights[key] = value++ for l in range(self.language_model.args.num_hidden_layers):+ prefix = f"language_model.model.layers.{l}.mlp"+ gate_up_key = f"{prefix}.experts.gate_up_proj"+ if gate_up_key in new_weights:+ gate_up = new_weights.pop(gate_up_key)+ mid = gate_up.shape[-2] // 2+ new_weights[f"{prefix}.switch_mlp.gate_proj.weight"] = gate_up[+ ..., :mid, :+ ]+ new_weights[f"{prefix}.switch_mlp.up_proj.weight"] = gate_up[+ ..., mid:, :+ ]+ new_weights[f"{prefix}.switch_mlp.down_proj.weight"] = new_weights.pop(+ f"{prefix}.experts.down_proj"+ )++ return self.language_model.sanitize(new_weights)diff --git a/src/edge0/backends/mlx/_impl/qwen3_next.py b/src/edge0/backends/mlx/_impl/qwen3_next.pynew file mode 100644index 0000000..5062513--- /dev/null+++ b/src/edge0/backends/mlx/_impl/qwen3_next.py@@ -0,0 +1,504 @@+# Copyright © 2025 Apple Inc.++from __future__ import annotations++from dataclasses import dataclass+from functools import partial+from typing import Any, Dict, List, Optional, Tuple, Union++import mlx.core as mx+import mlx.nn as nn+from mlx.nn.layers.distributed import sum_gradients++from mlx_lm.models.activations import swiglu+from mlx_lm.models.base import (+ BaseModelArgs,+ create_attention_mask,+ create_ssm_mask,+ scaled_dot_product_attention,+)+from mlx_lm.models.cache import ArraysCache, KVCache+from mlx_lm.models.gated_delta import gated_delta_update+from mlx_lm.models.rope_utils import initialize_rope+from mlx_lm.models.switch_layers import SwitchGLU+++@dataclass+class ModelArgs(BaseModelArgs):+ model_type: str+ hidden_size: int+ num_hidden_layers: int+ intermediate_size: int+ num_attention_heads: int+ linear_num_value_heads: int+ linear_num_key_heads: int+ linear_key_head_dim: int+ linear_value_head_dim: int+ linear_conv_kernel_dim: int+ num_experts: int+ num_experts_per_tok: int+ decoder_sparse_step: int+ shared_expert_intermediate_size: int+ mlp_only_layers: List[int]+ moe_intermediate_size: int+ rms_norm_eps: float+ vocab_size: int+ num_key_value_heads: int+ rope_theta: float+ partial_rotary_factor: float+ max_position_embeddings: int+ head_dim: int+ norm_topk_prob: bool = False+ tie_word_embeddings: bool = False+ attention_bias: bool = False+ rope_scaling: Optional[Dict[str, Union[float, str]]] = None+ full_attention_interval: int = 4+++@partial(mx.compile, shapeless=True)+def _precise_swiglu(h, gate, x):+ gate = nn.silu(gate.astype(mx.float32))+ x = x.astype(mx.float32)+ return (gate * x).astype(h.dtype)+++class Qwen3NextRMSNormGated(nn.Module):+ def __init__(self, hidden_size: int, eps: float = 1e-6):+ super().__init__()+ self.eps = eps+ self.weight = mx.ones(hidden_size)++ def __call__(+ self, hidden_states: mx.array, gate: mx.array | None = None+ ) -> mx.array:+ x = mx.fast.rms_norm(hidden_states, self.weight, self.eps)+ if gate is not None:+ return _precise_swiglu(hidden_states, gate, x)+ else:+ return x.astype(hidden_states.dtype)+++class Qwen3NextAttention(nn.Module):+ def __init__(self, args: ModelArgs):+ super().__init__()+ self.num_key_value_heads = args.num_key_value_heads+ self.num_attention_heads = args.num_attention_heads+ self.head_dim = args.head_dim+ self.scale = self.head_dim**-0.5++ self.q_proj = nn.Linear(+ args.hidden_size,+ self.num_attention_heads * self.head_dim * 2,+ bias=args.attention_bias,+ )+ self.k_proj = nn.Linear(+ args.hidden_size,+ self.num_key_value_heads * self.head_dim,+ bias=args.attention_bias,+ )+ self.v_proj = nn.Linear(+ args.hidden_size,+ self.num_key_value_heads * self.head_dim,+ bias=args.attention_bias,+ )+ self.o_proj = nn.Linear(+ self.num_attention_heads * self.head_dim,+ args.hidden_size,+ bias=args.attention_bias,+ )++ self.q_norm = nn.RMSNorm(self.head_dim, eps=args.rms_norm_eps)+ self.k_norm = nn.RMSNorm(self.head_dim, eps=args.rms_norm_eps)++ self.rope = initialize_rope(+ int(self.head_dim * args.partial_rotary_factor),+ base=args.rope_theta,+ traditional=False,+ scaling_config=args.rope_scaling,+ max_position_embeddings=args.max_position_embeddings,+ )++ def __call__(+ self,+ x: mx.array,+ mask: Optional[mx.array] = None,+ cache: Optional[Any] = None,+ ) -> mx.array:+ B, L, D = x.shape++ q_proj_output = self.q_proj(x)+ queries, gate = mx.split(+ q_proj_output.reshape(B, L, self.num_attention_heads, -1), 2, axis=-1+ )+ gate = gate.reshape(B, L, -1)++ keys, values = self.k_proj(x), self.v_proj(x)++ queries = self.q_norm(queries).transpose(0, 2, 1, 3)+ keys = self.k_norm(keys.reshape(B, L, self.num_key_value_heads, -1)).transpose(+ 0, 2, 1, 3+ )+ values = values.reshape(B, L, self.num_key_value_heads, -1).transpose(+ 0, 2, 1, 3+ )++ if cache is not None:+ queries = self.rope(queries, offset=cache.offset)+ keys = self.rope(keys, offset=cache.offset)+ keys, values = cache.update_and_fetch(keys, values)+ else:+ queries = self.rope(queries)+ keys = self.rope(keys)++ output = scaled_dot_product_attention(+ queries, keys, values, cache=cache, scale=self.scale, mask=mask+ )+ output = output.transpose(0, 2, 1, 3).reshape(B, L, -1)++ return self.o_proj(output * mx.sigmoid(gate))+++class Qwen3NextMLP(nn.Module):+ def __init__(self, dim, hidden_dim):+ super().__init__()+ self.gate_proj = nn.Linear(dim, hidden_dim, bias=False)+ self.down_proj = nn.Linear(hidden_dim, dim, bias=False)+ self.up_proj = nn.Linear(dim, hidden_dim, bias=False)++ def __call__(self, x) -> mx.array:+ return self.down_proj(swiglu(self.gate_proj(x), self.up_proj(x)))+++class Qwen3NextGatedDeltaNet(nn.Module):+ def __init__(self, config: ModelArgs):+ super().__init__()+ self.hidden_size = config.hidden_size+ self.num_v_heads = config.linear_num_value_heads+ self.num_k_heads = config.linear_num_key_heads+ self.head_k_dim = config.linear_key_head_dim+ self.head_v_dim = config.linear_value_head_dim+ self.key_dim = self.head_k_dim * self.num_k_heads+ self.value_dim = self.head_v_dim * self.num_v_heads+ if self.num_v_heads % self.num_k_heads != 0:+ raise ValueError(+ f"num_v_heads ({self.num_v_heads}) must be divisible by num_k_heads ({self.num_k_heads})"+ )++ self.conv_kernel_size = config.linear_conv_kernel_dim+ self.layer_norm_epsilon = config.rms_norm_eps++ self.conv_dim = self.key_dim * 2 + self.value_dim+ self.conv1d = nn.Conv1d(+ in_channels=self.conv_dim,+ out_channels=self.conv_dim,+ bias=False,+ kernel_size=self.conv_kernel_size,+ groups=self.conv_dim,+ padding=0,+ )++ self.in_proj_qkvz = nn.Linear(+ self.hidden_size, self.key_dim * 2 + self.value_dim * 2, bias=False+ )+ self.in_proj_ba = nn.Linear(self.hidden_size, self.num_v_heads * 2, bias=False)++ self.dt_bias = mx.ones(self.num_v_heads)++ A = mx.random.uniform(low=0, high=16, shape=(self.num_v_heads,))+ self.A_log = mx.log(A)++ self.norm = Qwen3NextRMSNormGated(self.head_v_dim, eps=self.layer_norm_epsilon)++ self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)++ def fix_query_key_value_ordering(+ self, mixed_qkvz: mx.array, mixed_ba: mx.array+ ) -> mx.array:+ nk, dn, nv, dv = (+ self.num_k_heads,+ self.head_k_dim,+ self.num_v_heads,+ self.head_v_dim,+ )+ mixed_qkvz = mixed_qkvz.reshape(*mixed_qkvz.shape[:-1], nk, -1)+ mixed_ba = mixed_ba.reshape(*mixed_ba.shape[:-1], nk, -1)+ q, k, v, z = mx.split(mixed_qkvz, [dn, 2 * dn, 2 * dn + nv // nk * dv], axis=-1)+ b, a = mx.split(mixed_ba, [nv // nk], axis=-1)+ return (+ q,+ k,+ v.reshape(*v.shape[:2], -1, dv),+ z.reshape(*z.shape[:2], -1, dv),+ b.reshape(*b.shape[:2], nv),+ a.reshape(*a.shape[:2], nv),+ )++ def __call__(+ self,+ inputs: mx.array,+ mask: Optional[mx.array] = None,+ cache: Optional[Any] = None,+ ) -> mx.array:+ B, S, _ = inputs.shape+ q, k, v, z, b, a = self.fix_query_key_value_ordering(+ self.in_proj_qkvz(inputs), self.in_proj_ba(inputs)+ )++ if cache is not None and cache[0] is not None:+ conv_state = cache[0]+ else:+ conv_state = mx.zeros(+ (B, self.conv_kernel_size - 1, self.conv_dim),+ dtype=inputs.dtype,+ )++ mixed_qkv = mx.concatenate(+ [q.reshape(B, S, -1), k.reshape(B, S, -1), v.reshape(B, S, -1)], axis=-1+ )+ if mask is not None:+ mixed_qkv = mx.where(mask[..., None], mixed_qkv, 0)+ conv_input = mx.concatenate([conv_state, mixed_qkv], axis=1)++ if cache is not None:+ n_keep = self.conv_kernel_size - 1+ if cache.lengths is not None:+ ends = mx.clip(cache.lengths, 0, S)+ positions = (ends[:, None] + mx.arange(n_keep))[..., None]+ cache[0] = mx.take_along_axis(conv_input, positions, axis=1)+ else:+ cache[0] = conv_input[:, -n_keep:, :]++ conv_out = nn.silu(self.conv1d(conv_input))++ q, k, v = [+ t.reshape(B, S, h, d)+ for t, h, d in zip(+ mx.split(conv_out, [self.key_dim, 2 * self.key_dim], -1),+ [self.num_k_heads, self.num_k_heads, self.num_v_heads],+ [self.head_k_dim, self.head_k_dim, self.head_v_dim],+ )+ ]++ state = cache[1] if cache else None+ inv_scale = k.shape[-1] ** -0.5+ q = (inv_scale**2) * mx.fast.rms_norm(q, None, 1e-6)+ k = inv_scale * mx.fast.rms_norm(k, None, 1e-6)++ out, state = gated_delta_update(+ q,+ k,+ v,+ a,+ b,+ self.A_log,+ self.dt_bias,+ state,+ mask,+ use_kernel=not self.training,+ )++ if cache is not None:+ cache[1] = state+ cache.advance(S)++ out = self.norm(out, z)+ return self.out_proj(out.reshape(B, S, -1))+++class Qwen3NextSparseMoeBlock(nn.Module):+ def __init__(self, args: ModelArgs):+ super().__init__()+ dim = args.hidden_size+ intermediate_size = args.moe_intermediate_size+ shared_expert_intermediate_size = args.shared_expert_intermediate_size++ self.norm_topk_prob = args.norm_topk_prob+ self.num_experts = num_experts = args.num_experts+ self.top_k = args.num_experts_per_tok++ self.gate = nn.Linear(dim, num_experts, bias=False)+ self.switch_mlp = SwitchGLU(dim, intermediate_size, num_experts)++ self.shared_expert = Qwen3NextMLP(dim, shared_expert_intermediate_size)+ self.shared_expert_gate = nn.Linear(dim, 1, bias=False)++ self.sharding_group = None++ def __call__(+ self,+ x: mx.array,+ ) -> mx.array:+ if self.sharding_group is not None:+ x = sum_gradients(self.sharding_group)(x)++ gates = self.gate(x)+ gates = mx.softmax(gates, axis=-1, precise=True)++ k = self.top_k+ inds = mx.argpartition(gates, kth=-k, axis=-1)[..., -k:]+ scores = mx.take_along_axis(gates, inds, axis=-1)+ if self.norm_topk_prob:+ scores = scores / scores.sum(axis=-1, keepdims=True)++ y = self.switch_mlp(x, inds)+ y = (y * scores[..., None]).sum(axis=-2)++ shared_y = self.shared_expert(x)+ shared_y = mx.sigmoid(self.shared_expert_gate(x)) * shared_y++ y = y + shared_y++ if self.sharding_group is not None:+ y = mx.distributed.all_sum(y, group=self.sharding_group)++ return y+++class Qwen3NextDecoderLayer(nn.Module):+ def __init__(self, args: ModelArgs, layer_idx: int):+ super().__init__()+ self.is_linear = (layer_idx + 1) % args.full_attention_interval != 0+ if self.is_linear:+ self.linear_attn = Qwen3NextGatedDeltaNet(args)+ else:+ self.self_attn = Qwen3NextAttention(args)++ self.input_layernorm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)+ self.post_attention_layernorm = nn.RMSNorm(+ args.hidden_size, eps=args.rms_norm_eps+ )+ if (layer_idx not in args.mlp_only_layers) and (+ args.num_experts > 0 and (layer_idx + 1) % args.decoder_sparse_step == 0+ ):+ self.mlp = Qwen3NextSparseMoeBlock(args)+ else:+ self.mlp = Qwen3NextMLP(args.hidden_size, args.intermediate_size)++ def __call__(+ self,+ x: mx.array,+ mask: Optional[mx.array] = None,+ cache: Optional[Any] = None,+ ) -> mx.array:+ if self.is_linear:+ r = self.linear_attn(self.input_layernorm(x), mask, cache)+ else:+ r = self.self_attn(self.input_layernorm(x), mask, cache)+ h = x + r+ out = h + self.mlp(self.post_attention_layernorm(h))+ return out+++class Qwen3NextModel(nn.Module):+ def __init__(self, args: ModelArgs):+ super().__init__()+ self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size)+ self.layers = [+ Qwen3NextDecoderLayer(args=args, layer_idx=i)+ for i in range(args.num_hidden_layers)+ ]+ self.norm = nn.RMSNorm(args.hidden_size, eps=args.rms_norm_eps)+ self.ssm_idx = 0+ self.fa_idx = args.full_attention_interval - 1++ def __call__(+ self,+ inputs: mx.array,+ cache: Optional[Any] = None,+ before_layer_cb=None,+ after_layer_cb=None,+ async_eval_per_layer: bool = False,+ ) -> mx.array:+ hidden_states = self.embed_tokens(inputs)++ if cache is None:+ cache = [None] * len(self.layers)++ fa_mask = create_attention_mask(hidden_states, cache[self.fa_idx])+ ssm_mask = create_ssm_mask(hidden_states, cache[self.ssm_idx])++ for li, (layer, c) in enumerate(zip(self.layers, cache)):+ if before_layer_cb is not None:+ before_layer_cb(li)+ mask = ssm_mask if layer.is_linear else fa_mask+ hidden_states = layer(hidden_states, mask=mask, cache=c)+ if after_layer_cb is not None:+ after_layer_cb(li, hidden_states)+ if async_eval_per_layer:+ # Fire this layer's graph on the GPU while the CPU keeps+ # building the next layer (overlaps the whole-layer weight+ # load of the E3b prefill path). The final logits eval+ # still waits for everything.+ mx.async_eval(hidden_states)++ return self.norm(hidden_states)+++class Model(nn.Module):+ def __init__(self, args: ModelArgs):+ super().__init__()+ self.args = args+ self.model_type = args.model_type+ self.model = Qwen3NextModel(args)+ if not args.tie_word_embeddings:+ self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False)++ def __call__(+ self,+ inputs: mx.array,+ cache: Optional[Any] = None,+ ) -> mx.array:+ out = self.model(inputs, cache)+ if self.args.tie_word_embeddings:+ out = self.model.embed_tokens.as_linear(out)+ else:+ out = self.lm_head(out)+ return out++ @property+ def layers(self):+ return self.model.layers++ def make_cache(self):+ return [ArraysCache(size=2) if l.is_linear else KVCache() for l in self.layers]++ def sanitize(self, weights):+ if "model.layers.0.mlp.experts.0.up_proj.weight" not in weights:+ return weights+ weights = {key: value for key, value in weights.items() if "mtp." not in key}++ if self.args.tie_word_embeddings:+ weights.pop("lm_head.weight", None)++ for l in range(self.args.num_hidden_layers):+ prefix = f"model.layers.{l}.mlp"+ for n in ["up_proj", "down_proj", "gate_proj"]:+ to_join = [+ weights.pop(f"{prefix}.experts.{e}.{n}.weight")+ for e in range(self.args.num_experts)+ ]+ weights[f"{prefix}.switch_mlp.{n}.weight"] = mx.stack(to_join)++ norm_keys = (+ ".input_layernorm.weight",+ ".post_attention_layernorm.weight",+ "model.norm.weight",+ ".q_norm.weight",+ ".k_norm.weight",+ )+ for k, v in weights.items():+ if "conv1d.weight" in k and v.shape[-1] != 1:+ weights[k] = v.moveaxis(2, 1)+ if any(k.endswith(sfx) for sfx in norm_keys):+ if v.ndim == 1:+ weights[k] = v + 1.0+ return weights++ @property+ def quant_predicate(self):+ def predicate(path, _):+ if path.endswith("mlp.gate") or path.endswith("shared_expert_gate"):+ return {"group_size": 64, "bits": 8}+ return True++ return predicatediff --git a/src/edge0/backends/mlx/backend.py b/src/edge0/backends/mlx/backend.pynew file mode 100644index 0000000..d4f52b7--- /dev/null+++ b/src/edge0/backends/mlx/backend.py@@ -0,0 +1,25 @@+"""MLX backend: namespace assembly.++``core`` is mlx.core (arrays, ops, eval/compile, random), ``nn`` is+mlx.nn (module factory), ``io`` and ``quant`` are edge0 wrappers around+mlx-lm and mlx's quantized gather kernels.+"""++from __future__ import annotations++import mlx.core as core # noqa: F401+import mlx.nn as nn # noqa: F401++from edge0.backends.mlx import io, quant # noqa: F401+++class BackendImpl:+ """Handle to the active backend implementation."""++ name = "mlx"+ description = "MLX / Metal on Apple Silicon"++ @property+ def version(self) -> str:+ import mlx+ return mlx.__version__diff --git a/src/edge0/backends/mlx/io.py b/src/edge0/backends/mlx/io.pynew file mode 100644index 0000000..242a4ea--- /dev/null+++ b/src/edge0/backends/mlx/io.py@@ -0,0 +1,155 @@+"""MLX backend: model / tokenizer / tensor-store loading."""++from __future__ import annotations++import json+import mmap+import struct++import numpy as np+import mlx.core as mx+from mlx_lm.utils import load_model as _load_model++from edge0.backends.base import TensorStore+++def load_model(model_path, lazy=True, strict=False, model_config=None,+ get_model_classes=None):+ """Load a checkpoint skeleton via mlx-lm.++ ``get_model_classes`` (callable config -> ``(Model, ModelArgs)``)+ selects a vendored class pair instead of mlx-lm's registry; its+ ``ModelArgs`` fields can be overridden through ``model_config`` —+ the hook used by the edge0 model adapters (e.g. prerouter wiring+ flags). ``lazy=True`` keeps weights as mmap-backed lazy arrays so+ the skeleton occupies no RAM (only the small non-MoE parts are ever+ materialized).+ """+ from pathlib import Path+ if not isinstance(model_path, Path):+ model_path = Path(model_path)+ return _load_model(model_path, lazy=lazy, strict=strict,+ model_config=model_config,+ get_model_classes=get_model_classes)+++def load_tokenizer(model_path):+ """Load a HuggingFace tokenizer from a LOCAL directory only.++ mlx-lm 0.31's ``load_tokenizer`` funnels the path through its+ ``_download`` (hf hub snapshot logic), which prompts for+ ``trust_remote_code`` when the checkpoint carries an ``auto_map`` —+ undesirable for a local-only serving stack. The transformers+ ``AutoTokenizer`` loads a local directory without any hub round trip.+ """+ from transformers import AutoTokenizer+ return AutoTokenizer.from_pretrained(+ model_path, local_files_only=True, trust_remote_code=True)+++def open_shards(model_dir: str) -> list:+ """Open every ``model*.safetensors`` shard as a byte-range mmap+ (qwen-style multi-shard checkpoints; a single-file checkpoint yields+ a one-element list)."""+ import glob+ import os+ from edge0.streaming.mmap import SafetensorsMmap+ shards = []+ for path in sorted(glob.glob(os.path.join(+ os.fspath(model_dir), "model*.safetensors"))):+ shards.append(SafetensorsMmap(path))+ if not shards:+ raise FileNotFoundError(+ f"no model*.safetensors shards under {model_dir}")+ return shards+++def load_safetensors(path: str, dtype=None) -> dict[str, "mx.array"]:+ """Load every tensor of a (small) safetensors file as mx arrays.++ Used for adapter / prerouter weight files; dtype converts (e.g.+ mx.float16) without copying when it matches the stored dtype.+ """+ import mlx.core as mx+ store = SafeTensorsStore(path)+ try:+ out = {}+ for name in store.keys():+ arr = store.get(name)+ if dtype is not None and arr.dtype != dtype:+ arr = arr.astype(dtype)+ out[name] = arr+ return out+ finally:+ store.close()+++class SafeTensorsStore(TensorStore):+ """mmap-backed safetensors store with per-tensor lazy access.++ Tensors are returned as ``mx.array`` views (zero copy). ``get`` never+ materializes the whole file.+ """++ def __init__(self, path: str):+ import mlx.core as mx+ self._mx = mx+ self._path = path+ with open(path, "rb") as f:+ header_len = struct.unpack("<Q", f.read(8))[0]+ header_bytes = f.read(header_len)+ header = json.loads(header_bytes)+ self._metadata = header.pop("__metadata__", {})+ self._entries = {+ name: {+ "offset": meta["data_offsets"][0] + 8 + header_len,+ "size": meta["data_offsets"][1] - meta["data_offsets"][0],+ "dtype": meta["dtype"],+ "shape": tuple(meta["shape"]),+ }+ for name, meta in header.items()+ }+ self._keys = sorted(self._entries)+ self._file = open(path, "rb")+ self._mm = mmap.mmap(self._file.fileno(), 0, access=mmap.ACCESS_READ)++ @property+ def path(self) -> str:+ return self._path++ def keys(self) -> list[str]:+ return list(self._keys)++ def metadata(self) -> dict:+ return dict(self._metadata)++ def get(self, name: str):+ e = self._entries.get(name)+ if e is None:+ raise KeyError(f"{self.path}: no tensor {name!r}")+ buf = np.frombuffer(self._mm, dtype=np.uint8, count=e["size"],+ offset=e["offset"])+ arr = np.frombuffer(buf, dtype=_DTYPES[e["dtype"]],+ count=int(np.prod(e["shape"])))+ return self._mx.array(arr.reshape(e["shape"]))++ def close(self):+ self._mm.close()+ self._file.close()++ def __del__(self):+ try:+ self.close()+ except Exception:+ pass+++# Some numpy builds lack ``np.bfloat16`` (absent on this machine's+# 2.5.3); mlx has a native bfloat16 we fall back to — the array comes+# back as an mlx array either way.+_BF16 = getattr(np, "bfloat16", None) or mx.bfloat16+_DTYPES = {+ "F64": np.float64, "F32": np.float32, "F16": np.float16,+ "BF16": _BF16, "I64": np.int64, "I32": np.int32,+ "I16": np.int16, "I8": np.int8, "U8": np.uint8, "BOOL": np.bool_,+}diff --git a/src/edge0/backends/mlx/quant.py b/src/edge0/backends/mlx/quant.pynew file mode 100644index 0000000..b1ca11d--- /dev/null+++ b/src/edge0/backends/mlx/quant.py@@ -0,0 +1,26 @@+"""MLX backend: quantized gather kernels (bit-compatible with mlx-lm)."""++from __future__ import annotations++import mlx.core as mx+++def gather_qmm(x, w, scales, biases, rhs_indices, transpose=True,+ group_size=64, bits=4, mode="affine",+ sorted_indices=False):+ """Quantized matmul over a gathered subset of expert rows.++ ``w`` is a stacked [num_experts, out, in] weight tensor; rows are+ gathered by ``rhs_indices`` before dequantizing. Same kernel and+ defaults as the resident (non-streaming) MoE path, so outputs are+ bit-identical to a resident ``SwitchGLU``.+ """+ return mx.gather_qmm(+ x, w, scales, biases, rhs_indices=rhs_indices, transpose=transpose,+ group_size=group_size, bits=bits, mode=mode,+ sorted_indices=sorted_indices)+++def swiglu(up: mx.array, gate: mx.array) -> mx.array:+ """SiLU gated activation: silu(gate) * up."""+ return mx.nn.silu(gate) * updiff --git a/src/edge0/cli.py b/src/edge0/cli.pynew file mode 100644index 0000000..0598d26--- /dev/null+++ b/src/edge0/cli.py@@ -0,0 +1,253 @@+"""edge0 command-line entry point (``edge0``).++Commands:++ edge0 demo one-command quickstart: find a checkpoint, generate+ edge0 serve start the HTTP server (one model, queued generations)+ edge0 chat one-shot prompt -> answer on the terminal+ edge0 models list registered tiers and their default profiles+ edge0 convert-adapters one-shot legacy npz -> safetensors migration+"""++from __future__ import annotations++import argparse+import os+import sys++from edge0 import models # noqa: F401 (populates MODEL_REGISTRY)+from edge0.registry import MODEL_REGISTRY++# Checkpoint locations the demo command probes when no --model-dir is+# given (the dev boxes these tiers were validated on).+DEMO_DEFAULTS = {+ "edge0-35b": "/Users/linyu/Documents/qwen35-v7-deploy/model",+ "edge0-10b": "/Users/linyu/Documents/ling-mlx-server/v7-deploy/model",+}++DEMO_PROMPTS = {+ "edge0-35b": "Hello! Write one short sentence about Zhuhai.",+ "edge0-10b": "你好,用一句话介绍珠海。",+}+++def cmd_models(args) -> int:+ for name in sorted(MODEL_REGISTRY):+ mod = MODEL_REGISTRY[name]+ cfg = mod.Config.from_pretrained(None) # tier defaults+ print(f"{name} (port {cfg.port}, target {cfg.target_tok_s} tok/s, "+ f"peak ≈ {cfg.peak_active_mem_mb:.0f} MB)")+ print(f" experts={cfg.moe_spec.num_experts} "+ f"top_k={cfg.moe_spec.top_k} "+ f"quant={cfg.moe_spec.quant.bits}bit/"+ f"g{cfg.moe_spec.quant.group_size} "+ f"staged_n={cfg.options.staged_n} "+ f"prefill_full={cfg.options.prefill_full_layers} "+ f"hot={cfg.options.hot_per_layer}")+ pr = cfg.prerouter+ if pr is not None:+ owners = pr.owners+ owners_txt = (f"owners={owners[0]}..{owners[-1]} ({len(owners)})"+ if owners else "owners=default")+ print(f" prerouter: start={pr.start_layer} hidden={pr.hidden} "+ f"dtype={pr.dtype} K={cfg.prerouter_top_k} {owners_txt} "+ f"weights={pr.weights_file}")+ if cfg.lora:+ print(f" lora: {cfg.lora} (r={cfg.lora_r} alpha={cfg.lora_alpha})")+ return 0+++def _engine_kwargs(args) -> dict:+ """Map CLI flags to from_pretrained overrides (absent key = keep the+ tier default; explicit None/' ' disables)."""+ kw: dict = {}+ if getattr(args, "no_prerouter", False):+ kw["prerouter"] = None+ if getattr(args, "no_lora", False):+ kw["lora"] = ""+ return kw+++def _resolve_model(args) -> tuple[str | None, str | None]:+ """vLLM-style model argument resolution.++ ``edge0 serve <model>`` accepts either a registered tier name+ (``edge0-35b`` / ``edge0-10b`` — checkpoint found via the matching+ env var, else the known dev-box default) or a checkpoint path+ (``edge0 serve /root/edge0-35b`` — tier auto-detected from the+ checkpoint's ``config.json``).++ Returns ``(model_dir, name)``; either may stay None to keep the+ legacy ``--model-dir`` / ``--name`` behaviour.+ """+ model = getattr(args, "model", None)+ if not model:+ return args.model_dir, args.name+ if model in MODEL_REGISTRY:+ short = (model.split("-", 1)[-1] if model.startswith("edge0-")+ else model)+ env = os.environ.get(f"EDGE0_{short.upper()}_MODEL")+ return env or DEMO_DEFAULTS.get(model), model+ return model, None # a path: tier auto-detected by the registry+++def cmd_demo(args) -> int:+ from edge0 import AutoEngine+ from edge0.server.chat import ChatMessage, ChatRequest, ChatSession++ model_dir, name = _resolve_model(args)+ if not model_dir:+ # No explicit model: probe the known dev-box checkpoints.+ for d in DEMO_DEFAULTS.values():+ if os.path.isdir(d):+ model_dir = d+ break+ if not model_dir or not os.path.isdir(model_dir):+ print(+ f"[edge0] no checkpoint for {name or 'model'}.\n"+ "Pass the model explicitly, vLLM-style:\n"+ f" edge0 demo {DEMO_DEFAULTS.get(name, '<checkpoint-dir>')}\n"+ "or point at your own checkpoint:\n"+ " edge0 demo /path/to/qwen35/model\n"+ "Tier names are edge0-35b / edge0-10b (env EDGE0_<TIER>_MODEL "+ "overrides the default search path).",+ file=sys.stderr,+ )+ return 2+ engine = AutoEngine.from_pretrained(model_dir, name=name,+ **_engine_kwargs(args))+ tok = engine._tok+ if tok is None:+ engine.close()+ raise SystemExit("model has no tokenizer; cannot demo")+ prompt = args.prompt or DEMO_PROMPTS.get(engine.name, "Hello!")+ req = ChatRequest(model=engine.name, messages=[+ ChatMessage(role="user", content=prompt)])+ sess = ChatSession(engine, req)+ tokens, meta = sess.run()+ print(f"user : {prompt}")+ print(f"edge0: {tok.decode(tokens)}")+ print(f"# {len(tokens)} tokens in {meta['wall_s']}s",+ file=sys.stderr)+ engine.close()+ return 0+++def cmd_chat(args) -> int:+ from edge0 import AutoEngine++ model_dir, name = _resolve_model(args)+ if not model_dir or not os.path.isdir(model_dir):+ raise SystemExit(+ "chat requires a checkpoint; pass it as the model argument "+ "(edge0 chat /path/to/model) or --model-dir")+ engine = AutoEngine.from_pretrained(model_dir, name=name,+ **_engine_kwargs(args))+ tok = engine._tok+ if tok is None:+ raise SystemExit("model has no tokenizer; cannot chat")+ if args.prompt:+ prompts = [args.prompt]+ elif not sys.stdin.isatty():+ prompts = [line.rstrip("\n") for line in sys.stdin if line.strip()]+ else:+ raise SystemExit("pass --prompt or pipe input on stdin")+ for p in prompts:+ from edge0.server.chat import ChatMessage, ChatRequest, ChatSession+ req = ChatRequest(model=name, messages=[+ ChatMessage(role="user", content=p)])+ sess = ChatSession(engine, req)+ tokens, meta = sess.run()+ print(tok.decode(tokens))+ print(f"# {len(tokens)} tokens in {meta['wall_s']}s", file=sys.stderr)+ engine.close()+ return 0+++def cmd_serve(args) -> int:+ from edge0 import AutoEngine+ from edge0.server import QueueServer, run_server++ model_dir, name = _resolve_model(args)+ if not model_dir or not os.path.isdir(model_dir):+ raise SystemExit(+ "serve requires a checkpoint; pass it as the model argument "+ "(edge0 serve /path/to/model) or --model-dir")+ engine = AutoEngine.from_pretrained(model_dir, name=name,+ **_engine_kwargs(args))+ server = QueueServer(engine, model_name=name or engine.name)+ print(f"[edge0] serving {server.model_name} on http://{args.host}:{args.port} "+ f"(stream={'flask' if args.flask else 'stdlib'})",+ file=sys.stderr)+ run_server(server, host=args.host, port=args.port, use_flask=args.flask)+ return 0+++def cmd_convert(args) -> int:+ import runpy+ sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve()+ .parents[1] / "scripts"))+ runpy.run_path(+ str(__import__("pathlib").Path(__file__).resolve().parents[1]+ / "scripts" / "convert_adapters_legacy.py"),+ run_name="__main__",+ )+ return 0+++def main(argv: list[str] | None = None) -> int:+ ap = argparse.ArgumentParser(prog="edge0", description=__doc__)+ sub = ap.add_subparsers(dest="cmd", required=True)++ p = sub.add_parser("models", help="list registered model tiers")+ p.set_defaults(fn=cmd_models)++ p = sub.add_parser(+ "demo",+ help="one-command quickstart (vLLM-style model argument)")+ p.add_argument("model", nargs="?", default=None,+ help="tier name (edge0-35b) or checkpoint dir")+ p.add_argument("--model-dir", default=None)+ p.add_argument("--name", default=None)+ p.add_argument("--prompt", default=None)+ p.add_argument("--no-prerouter", action="store_true")+ p.add_argument("--no-lora", action="store_true")+ p.set_defaults(fn=cmd_demo)++ p = sub.add_parser(+ "chat",+ help="one-shot prompt answering (vLLM-style model argument)")+ p.add_argument("model", nargs="?", default=None,+ help="tier name (edge0-35b) or checkpoint dir")+ p.add_argument("--model-dir", default=None)+ p.add_argument("--name", default=None)+ p.add_argument("--prompt", default=None)+ p.add_argument("--no-prerouter", action="store_true")+ p.add_argument("--no-lora", action="store_true")+ p.set_defaults(fn=cmd_chat)++ p = sub.add_parser(+ "serve",+ help="run the HTTP server, vLLM-style: edge0 serve <model>")+ p.add_argument("model", nargs="?", default=None,+ help="tier name (edge0-35b) or checkpoint dir")+ p.add_argument("--model-dir", default=None)+ p.add_argument("--name", default=None)+ p.add_argument("--host", default="127.0.0.1")+ p.add_argument("--port", type=int, default=8000)+ p.add_argument("--flask", action="store_true",+ help="use the Flask transport (needs flask installed)")+ p.add_argument("--no-prerouter", action="store_true")+ p.add_argument("--no-lora", action="store_true")+ p.set_defaults(fn=cmd_serve)++ p = sub.add_parser("convert-adapters",+ help="one-shot legacy npz -> safetensors migration")+ p.set_defaults(fn=cmd_convert)++ args = ap.parse_args(argv)+ return args.fn(args)+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/src/edge0/config.py b/src/edge0/config.pynew file mode 100644index 0000000..f7cd6e3--- /dev/null+++ b/src/edge0/config.py@@ -0,0 +1,37 @@+"""Generation / sampling configuration (transformers-style)."""++from __future__ import annotations++from dataclasses import dataclass, field+++@dataclass+class GenerationConfig:+ """Sampling knobs for one model tier.++ Attributes:+ temperature: softmax temperature (0 disables).+ top_p: nucleus truncation (<1.0 enables).+ top_k: top-k truncation (>0 enables).+ repetition_penalty: HF-style penalty applied to the history of+ generated tokens.+ max_new_tokens: default cap for generated tokens.+ eos_ids: token ids that end generation.+ seed: optional RNG seed.+ first_token_greedy: always argmax the FIRST generated token+ (qwen35-v7-deploy serve.py / THINK_FIX_20260907.md pattern):+ after the think opener a randomly-sampled first token can+ derail the whole block into '!' loops.+ """++ temperature: float = 0.7+ top_p: float = 0.95+ top_k: int = 64+ repetition_penalty: float = 1.0+ max_new_tokens: int = 512+ eos_ids: tuple[int, ...] = ()+ seed: int | None = None+ first_token_greedy: bool = True++ def is_eos(self, token_id: int) -> bool:+ return token_id in self.eos_idsdiff --git a/src/edge0/engine/__init__.py b/src/edge0/engine/__init__.pynew file mode 100644index 0000000..56a6161--- /dev/null+++ b/src/edge0/engine/__init__.py@@ -0,0 +1,25 @@+"""edge0 engines: streaming prefill/decode lifecycles per model family.++``edge0.engine.base.Edge0Engine`` owns the shared prefill / step /+generate loops; ``edge0.engine.qwen.Qwen35Engine`` and+``edge0.engine.ling.Ling10BEngine`` implement the family hooks+(streaming install, prerouter wiring, step-boundary staging).+"""++from edge0.engine.base import Edge0Engine+from edge0.engine.hooks import (+ make_history_prefetch,+ make_intra_after_layer,+ make_prefill_before_layer,+)+from edge0.engine.ling import Ling10BEngine+from edge0.engine.qwen import Qwen35Engine++__all__ = [+ "Edge0Engine",+ "Qwen35Engine",+ "Ling10BEngine",+ "make_history_prefetch",+ "make_intra_after_layer",+ "make_prefill_before_layer",+]diff --git a/src/edge0/engine/base.py b/src/edge0/engine/base.pynew file mode 100644index 0000000..d13a22e--- /dev/null+++ b/src/edge0/engine/base.py@@ -0,0 +1,177 @@+"""Edge0 engine base: lifecycle + shared prefill / decode loops.++Family engines (``edge0.engine.qwen`` / ``edge0.engine.ling``) implement+the model-specific hooks; everything here is model-agnostic:++* ``prefill`` — chunked prompt processing; the family hook decides+ whole-layer (E3b) vs on-demand paths per chunk, and ``_prefill_end``+ drops the prefill working set, refreshes hot pins and primes the+ staged slots for the first decode step.+* ``step`` — one-token forward + the family's step-boundary staging+ (early submit for router layers, ``sync_actuals``, prerouter+ ``stage_all`` + state swap).+* ``generate`` — sampling loop with the family ``GenerationConfig``.++Loop mechanics ported from the deployment engines (``engine_qwen.py`` /+ling ``engine.py``) so the measured production behavior — one cheap sync+point per step, fills overlapping the next forward — is preserved.+"""++from __future__ import annotations++import time++from edge0.backends import core++from edge0.config import GenerationConfig+from edge0.sampling import sample+++class Edge0Engine:+ """Base streaming engine. Subclasses fill in the family hooks."""++ name = "edge0"++ def __init__(self, model_dir: str, cfg, tokenizer=None):+ self.dir = model_dir+ self.cfg = cfg+ self._tok = tokenizer+ self.prefill_chunk = getattr(cfg, "prefill_chunk", 2048)+ self.pos = 0+ self._prefill_active = False+ self._last_logits = None+ self._cap_mlx_cache()+ t0 = time.time()+ self._build()+ print(f"[{self.name}] built in {time.time() - t0:.1f}s", flush=True)++ # ---- family hooks -----------------------------------------------------++ def _build(self):+ raise NotImplementedError++ def _forward(self, ids, intra_stage: bool = True) -> core.array:+ raise NotImplementedError++ def _step_pre(self, token_id: int) -> None:+ """Called before a decode forward (early-submit fills)."""++ def _step_post(self, token_id: int) -> None:+ """Called after a decode forward's logits eval (staging/swap)."""++ def _prefill_end(self) -> None:+ """Called after the last prefill chunk."""++ def _reset_state(self) -> None:+ pass++ def _lm_logits(self, h: core.array) -> core.array:+ raise NotImplementedError++ # ---- shared loops -----------------------------------------------------++ def prefill(self, token_ids, chunk_size=None, on_progress=None):+ if chunk_size is None:+ chunk_size = self.prefill_chunk+ total = len(token_ids)+ done = 0+ self._prefill_active = True+ try:+ for start in range(0, total, chunk_size):+ chunk = token_ids[start:start + chunk_size]+ self._last_logits = self._forward(chunk)+ self.pos += len(chunk)+ done += len(chunk)+ if on_progress is not None:+ on_progress(done, total)+ finally:+ self._prefill_active = False+ self._prefill_end()+ return total++ def next_logits(self) -> core.array:+ return self._last_logits++ def step(self, token_id: int) -> core.array:+ self._step_pre(token_id)+ logits = self._forward([token_id])+ self.pos += 1+ self._step_post(token_id)+ return logits++ def generate(self, token_ids, gen_config: GenerationConfig | None = None,+ max_new_tokens: int | None = None, on_token=None) -> list[int]:+ """Prefill ``token_ids`` then sample until EOS or the cap."""+ if gen_config is None:+ gen_config = getattr(self.cfg, "gen", GenerationConfig())+ if max_new_tokens is None:+ max_new_tokens = gen_config.max_new_tokens+ if len(token_ids) > 1:+ self.prefill(token_ids)+ elif len(token_ids) == 1:+ self._last_logits = self._forward(token_ids)+ self.pos += 1+ out: list[int] = []+ history = list(token_ids)+ logits = self.next_logits()+ first_token = True+ for _ in range(max_new_tokens):+ if logits is None:+ raise RuntimeError("no logits available before first step")+ if first_token and gen_config.first_token_greedy:+ # The FIRST token is always greedy (qwen35-v7-deploy+ # serve.py pattern, THINK_FIX_20260907.md §3.2): after the+ # think opener a randomly-sampled first token can derail+ # the whole block into '!' loops.+ tid = int(core.argmax(logits, axis=-1).item())+ else:+ tid = sample(+ logits,+ temperature=gen_config.temperature,+ top_k=gen_config.top_k,+ top_p=gen_config.top_p,+ repetition_penalty=gen_config.repetition_penalty,+ history=history,+ seed=gen_config.seed,+ )+ first_token = False+ if gen_config.is_eos(tid):+ break+ out.append(tid)+ history.append(tid)+ # The deployment's step() returns the next logits and does NOT+ # update _last_logits (engine_qwen.py:471-474), so the loop+ # must carry the step return value — re-reading next_logits()+ # here would replay the prefill-final logits forever (the+ # greedy repetition bug: [353]*8 instead of [353, 2688, ...]).+ logits = self.step(tid)+ if on_token is not None:+ on_token(tid)+ return out++ def reset(self):+ self._reset_state()+ self.pos = 0+ self._last_logits = None++ def stats(self) -> dict:+ layers = getattr(self, "_all_stream_layers", {})+ return {li: exp.stats() for li, exp in layers.items()}++ def close(self):+ for exp in getattr(self, "_all_stream_layers", {}).values():+ try:+ exp.close()+ except Exception: # noqa: BLE001 — best-effort shutdown+ pass++ # ---- misc -------------------------------------------------------------++ @staticmethod+ def _cap_mlx_cache():+ import os+ try:+ mb = int(os.environ.get("MLX_CACHE_LIMIT_MB", "256"))+ core.set_cache_limit(mb * 1024 * 1024)+ except Exception: # noqa: BLE001 — best-effort+ passdiff --git a/src/edge0/engine/hooks.py b/src/edge0/engine/hooks.pynew file mode 100644index 0000000..c075dc5--- /dev/null+++ b/src/edge0/engine/hooks.py@@ -0,0 +1,87 @@+"""Prefill / decode callback factories.++All callbacks are ports of the deployment engines' proven hooks+(``engine_qwen.py`` / ling ``engine.py``); each is a pure function+building a callback from a ``dict[int, StreamingSwitchGLU]`` so the+same machinery serves both model families.+"""++from __future__ import annotations+++def make_prefill_before_layer(all_stream_layers, *, full_n: int = 0,+ hot_n: int = 0, hot_window: int = 4):+ """Before-layer prefill hook: whole-layer load-drop (E3b) + the+ sliding hot-expert window.++ Before layer ``li`` runs: drop layer ``li-1``'s whole-layer set (its+ async_eval was already submitted, so the GPU queue keeps the arrays+ alive until evaluated), then either load layer ``li``'s full stacked+ tensors (``full_n`` unset / ``li < full_n``) or build the hot-stack+ window (``hot_n``): numpy backing for layers ``li..li+ahead``,+ GPU materialization for the window, dematerialize trailing layers.+ """+ w = max(1, hot_window)+ ahead = max(1, w // 2)++ def before_layer(li: int) -> None:+ exp_old = all_stream_layers.get(li - 1)+ if exp_old is not None:+ exp_old.clear_full_layer()+ if full_n and li >= full_n:+ return+ if hot_n:+ for lj in range(li, min(li + ahead + 1,+ len(all_stream_layers))):+ exp_w = all_stream_layers.get(lj)+ if exp_w is None:+ continue+ exp_w.load_hot_layer(hot_n) # numpy backing (cheap)+ exp_w.materialize_hot() # GPU window+ for lj in range(0, max(0, li - (w - ahead))):+ exp_t = all_stream_layers.get(lj)+ if exp_t is not None:+ exp_t.dematerialize_hot()+ if not full_n:+ return+ exp = all_stream_layers.get(li)+ if exp is not None:+ exp.load_full_layer()++ return before_layer+++def make_intra_after_layer(all_stream_layers, enabled: bool):+ """Intra-step staging: right after layer ``li`` runs, submit the next+ step's layer-``li`` fill from ``last_used`` (the router's actuals from+ the previous step, already on the CPU — no mid-forward tolist, which+ segfaults in mlx 0.30.4 when a pool thread forces a GPU sync while the+ main thread is still building the graph)."""+ if not enabled:+ return None++ def after_layer(li: int, hidden_states=None) -> None:+ exp = all_stream_layers.get(li)+ if exp is None or not exp.last_used:+ return+ exp.stage_experts(list(exp.last_used))++ return after_layer+++def make_history_prefetch(all_stream_layers, enabled: bool):+ """Zero-cost decode prefetch: submit each layer's previous-token+ expert set (``last_used``) so the SSD loads overlap this step's GPU+ compute. The no-prerouter replacement for next-token prediction —+ exploits adjacent-token expert locality (~0.4 overlap measured)."""+ if not enabled:+ return None++ def history_prefetch() -> None:+ for li in range(1, len(all_stream_layers)):+ e = all_stream_layers.get(li)+ if e is None or not e.last_used:+ continue+ e.prefetch(list(e.last_used))++ return history_prefetchdiff --git a/src/edge0/engine/ling.py b/src/edge0/engine/ling.pynew file mode 100644index 0000000..b0f3936--- /dev/null+++ b/src/edge0/engine/ling.py@@ -0,0 +1,246 @@+"""edge0-10b engine: Ling 3.0 hybrid (MLA + MoE, 10B tier).++Port of the deployment's ling ``engine.py`` hybrid-prerouter path:++* the vendored ``bailing_hybrid.py`` model already owns its prerouter+ heads and consumes ``prerouter_cache`` logits inside+ ``BailingSparseMoE``'s ``_select_from_logits`` (sigmoid + group-limited+ top-k, routed-scaling 2.5) — the engine only feeds the cache and+ submits the staged fills.+* step boundary: ONE stacked head batch -> ONE tolist -> per-consumer+ ``stage_experts`` + ``pg_cache[consumer]`` logits; the next forward's+ consuming MoE blocks re-select from those cached logits, so staged+ set == routing set (zero drops) by construction.+* prefill — E3b whole-layer load-drop (``prefill_full_layers``).+"""++from __future__ import annotations++import os+from types import SimpleNamespace++from edge0.backends import core++from edge0.backends.mlx._impl.bailing_hybrid import Model as BailingModel+from edge0.backends.mlx._impl.bailing_hybrid import ModelArgs as BailingArgs+from edge0.backends import io+from edge0.backends.mlx.io import load_model, load_tokenizer+from edge0.engine.base import Edge0Engine+from edge0.engine.hooks import (+ make_history_prefetch,+ make_prefill_before_layer,+)+from edge0.prerouter.install import install_prerouter+from edge0.prerouter.stager import LingPrerouterStager+from edge0.streaming.install import install_streaming_experts+from edge0.streaming.mmap import SafetensorsMmap+++def _get_model_classes(config):+ """mlx-lm class hook: serve the vendored bailing backbone."""+ return BailingModel, BailingArgs+++def load_installed(model_dir: str, cfg):+ """Load the ling skeleton and install streaming twins, LoRA and the+ hybrid prerouter weights (shared by ``build_model``/``build_engine``).++ Returns ``(model, model_config, shards, installs)``.+ """+ model, model_config = load_model(+ model_dir, lazy=True, strict=False,+ model_config={"model_type": "bailing_hybrid",+ "prerouter_enabled": cfg.prerouter is not None,+ "prerouter_start_layer":+ getattr(cfg.prerouter, "start_layer", 7),+ "prerouter_hidden":+ getattr(cfg.prerouter, "hidden", 512)},+ get_model_classes=_get_model_classes)+ shards = [SafetensorsMmap(os.path.join(+ os.fspath(model_dir), "model.safetensors"))]+ spec = cfg.moe_spec+ opts = cfg.options+ n_layers = model_config["num_hidden_layers"]+ installed = install_streaming_experts(+ model, shards, spec, options=opts, num_layers=n_layers)+ all_stream = {li: t for li, t in enumerate(installed) if t is not None}+ stream_layers = {li: t for li, t in all_stream.items() if t._staged_mode}++ if cfg.lora:+ from edge0.adapters.lora import install_lora+ install_lora(model, cfg.lora, r=cfg.lora_r, alpha=cfg.lora_alpha)++ pg_state = pg_stager = None+ if cfg.prerouter and cfg.prerouter.weights_file:+ pg_state, heads = install_prerouter(+ model=model, spec=spec, pspec=cfg.prerouter, n_layers=n_layers)+ pg_stager = LingPrerouterStager(+ model=model, spec=spec, pspec=cfg.prerouter,+ state=pg_state, stream_layers=stream_layers,+ top_k=cfg.prerouter_top_k)+ print(f"[edge0-10b] prerouter installed: {len(heads)} heads, "+ f"start={cfg.prerouter.start_layer}, "+ f"K={cfg.prerouter_top_k}", flush=True)+ installs = dict(all_stream_layers=all_stream,+ stream_layers=stream_layers,+ pg_state=pg_state, pg_stager=pg_stager)+ return model, model_config, shards, installs+++class Ling10BEngine(Edge0Engine):+ """Streaming Ling-3.0 engine (staged decode + hybrid prerouter)."""++ name = "edge0-10b"++ def __init__(self, model_dir: str, cfg, tokenizer=None,+ think: bool = False):+ # Deployment parity: the ling server's THINK_MODE knob becomes the+ # engine's ``think`` flag (start_server.sh exports THINK_MODE=0).+ self.think = think+ self._chat_tpl = None+ # NAN_BANG_COLLAPSE_FIX.md parity: hidden clip default 1000 unless+ # the deployer overrides LING_HIDDEN_CLIP explicitly. One fp16+ # overflow inside a layer otherwise poisons the whole net into+ # all-NaN logits -> argmax fallback token 0 ('!') death spiral.+ if "LING_HIDDEN_CLIP" not in os.environ:+ os.environ["LING_HIDDEN_CLIP"] = "1000"+ super().__init__(model_dir, cfg, tokenizer=tokenizer)++ def _build(self):+ cfg = self.cfg+ (self.model, self.model_config, self.shards,+ inst) = load_installed(cfg.model_dir, cfg)+ self._all_stream_layers = inst["all_stream_layers"]+ self._stream_layers = inst["stream_layers"]+ self._pg_state = inst["pg_state"]+ self._pg_stager = inst["pg_stager"]+ opts = cfg.options+ if self._tok is None:+ try:+ self._tok = load_tokenizer(cfg.model_dir)+ except Exception: # noqa: BLE001 — tokenizer optional for CLI+ pass++ self._prefill_before_layer = make_prefill_before_layer(+ self._all_stream_layers,+ full_n=getattr(opts, "prefill_full_layers", 0),+ hot_n=0, hot_window=1)+ self._history_prefetch = make_history_prefetch(+ self._all_stream_layers, enabled=cfg.prefetch_history)++ self.cache = self.model.make_cache()++ # ---- forward ----------------------------------------------------------++ def _forward(self, ids, intra_stage: bool = True) -> core.array:+ inputs = core.array(ids)[None, :]+ opts = self.cfg.options+ prefill_multi = len(ids) > 1 and self._prefill_active+ full_layer = bool(+ prefill_multi and self._all_stream_layers+ and opts.full_layer_prefill)+ h = self.model.model(+ inputs, cache=self.cache,+ before_layer_cb=self._prefill_before_layer if prefill_multi+ else None,+ after_layer_cb=None,+ async_eval_per_layer=bool(prefill_multi and full_layer),+ prerouter_cache=(self._pg_stager.pg_cache+ if self._pg_stager is not None else None))+ logits = self.model.lm_head(h[0, -1])+ core.eval(logits)+ # Step boundary: ONE stacked head batch -> ONE tolist -> fills.+ if (self._pg_stager is not None and not self._prefill_active+ and len(ids) == 1):+ self._pg_stager.stage_all()+ return logits++ # ---- step hooks -------------------------------------------------------++ def _step_pre(self, token_id: int) -> None:+ if self._pg_stager is None and self._history_prefetch is not None:+ self._history_prefetch()++ def _step_post(self, token_id: int) -> None:+ for exp in self._stream_layers.values():+ exp.sync_actuals()+ if self._pg_stager is None:+ # History staging: each layer's next set is its previous+ # actuals (adjacent-token expert locality).+ for exp in self._stream_layers.values():+ if exp.last_used:+ exp.stage_experts(list(exp.last_used))+ exp.swap_staged()++ # ---- prefill / reset --------------------------------------------------++ def _prefill_end(self) -> None:+ for exp in self._all_stream_layers.values():+ exp.clear_full_layer()+ if self._pg_stager is not None:+ # Deployment parity: prefill's tail stages the FIRST decode+ # token's expert sets (engine.py:919 calls stage_all right+ # after prefill). Without this the first decode step has no+ # prerouter logits, falls back to the true gate for one step,+ # and the whole prediction chain shifts by one token+ # (observed: [23982, 2862, ...] vs correct [23982, 4264, ...]).+ self._pg_stager.stage_all()+ if self._stream_layers:+ for exp in self._stream_layers.values():+ exp.stage_from_prefill()++ def _reset_state(self) -> None:+ self.cache = self.model.make_cache()+ if self._pg_stager is not None:+ self._pg_stager.reset()+ self._pg_state.reset()++ def _lm_logits(self, h: core.array) -> core.array:+ return self.model.lm_head(h[0, -1])++ # ---- chat template (deployment parity) -------------------------------++ def _chat_template(self):+ """Render the deployment's ``chat_template.jinja`` (same file and+ renderer as the ling server's ``encode_chat``, engine.py)."""+ from jinja2 import BaseLoader, Environment, StrictUndefined++ if self._chat_tpl is None:+ src = open(+ os.path.join(self.dir, "chat_template.jinja"),+ encoding="utf-8",+ ).read()+ self._chat_tpl = Environment(+ loader=BaseLoader(), undefined=StrictUndefined,+ autoescape=False,+ ).from_string(src)+ return self._chat_tpl++ def encode_chat(self, messages, think=None) -> list:+ """Tokenize chat messages with the deployment chat template.++ ``think`` mirrors THINK_MODE: True renders "detailed thinking on"+ (the model answers with a reasoning preamble), False renders+ "detailed thinking off" (direct answer). Defaults to the+ engine's ``think`` flag.+ """+ if think is None:+ think = self.think+ msgs = [+ SimpleNamespace(+ role=m.get("role"),+ content=m.get("content") or "",+ reasoning_content=m.get("reasoning_content") or "",+ tool_calls=m.get("tool_calls"),+ )+ for m in messages+ ]+ text = self._chat_template().render(+ messages=msgs,+ add_generation_prompt=True,+ enable_thinking=bool(think),+ tools=None,+ )+ # transformers encode would prepend/append special tokens by+ # default; the template text is already complete.+ return self._tok.encode(text, add_special_tokens=False)diff --git a/src/edge0/engine/qwen.py b/src/edge0/engine/qwen.pynew file mode 100644index 0000000..75286cd--- /dev/null+++ b/src/edge0/engine/qwen.py@@ -0,0 +1,189 @@+"""edge0-35b engine: Qwen3.5-MoE (K=4 tier).++Port of the deployment's ``engine_qwen.py`` trained-prerouter path:++* prefill — E3b whole-layer load-drop (with the sliding hot-expert+ window), driven by the callback hooks added to the vendored+ ``qwen3_next.py``.+* decode — the prerouter head (class-level block patch) supplies the+ routing; at the step boundary ONE stacked head batch is evaluated,+ ONE tolist extracts every layer's next expert set, and the staged+ fills overlap the next forward (0 drops: staged set == routing set).+* router layers (< start, >= n-1) are staged from their actuals.+"""++from __future__ import annotations++from edge0.backends import core++from edge0.backends.mlx._impl.qwen3_5_moe import Model as Qwen35Model+from edge0.backends.mlx._impl.qwen3_5_moe import ModelArgs as Qwen35Args+from edge0.backends import io+from edge0.backends.mlx.io import load_model, load_tokenizer, open_shards+from edge0.engine.base import Edge0Engine+from edge0.engine.hooks import (+ make_history_prefetch,+ make_intra_after_layer,+ make_prefill_before_layer,+)+from edge0.prerouter.install import install_prerouter+from edge0.prerouter.stager import CrossTokenStager+from edge0.streaming.install import install_streaming_experts+++def _get_model_classes(config):+ """mlx-lm class hook: serve the vendored qwen3_5_moe backbone."""+ return Qwen35Model, Qwen35Args+++def load_installed(model_dir: str, cfg):+ """Load the qwen skeleton and install streaming twins, LoRA and the+ trained prerouter (shared by ``build_model`` and ``build_engine``).++ Returns ``(model, model_config, shards, installs)`` where+ ``installs`` carries the layer maps and prerouter state the engine+ drives at the step boundary.+ """+ model, model_config = load_model(+ model_dir, lazy=True, strict=False,+ model_config={"model_type": "qwen3_5_moe"},+ get_model_classes=_get_model_classes)+ shards = open_shards(model_dir)+ spec = cfg.moe_spec+ opts = cfg.options+ # qwen config.json nests the text params under ``text_config``; mlx-lm+ # returns the raw dict, so resolve defensively.+ _tc = model_config.get("text_config", model_config)+ n_layers = int(_tc.get("num_hidden_layers", 40))+ installed = install_streaming_experts(+ model, shards, spec, options=opts, num_layers=n_layers)+ all_stream = {li: t for li, t in enumerate(installed) if t is not None}+ stream_layers = {li: t for li, t in all_stream.items() if t._staged_mode}++ if cfg.lora:+ from edge0.adapters.lora import install_lora+ install_lora(model, cfg.lora, r=cfg.lora_r, alpha=cfg.lora_alpha)++ pg_state = pg_stager = None+ if cfg.prerouter and cfg.prerouter.weights_file:+ pg_state, heads = install_prerouter(+ model=model, spec=spec, pspec=cfg.prerouter, n_layers=n_layers)+ pg_stager = CrossTokenStager(+ model=model, spec=spec, pspec=cfg.prerouter,+ state=pg_state, stream_layers=stream_layers,+ top_k=cfg.prerouter_top_k)+ print(f"[edge0-35b] prerouter installed: {len(heads)} heads, "+ f"start={cfg.prerouter.start_layer}, "+ f"K={cfg.prerouter_top_k}", flush=True)+ installs = dict(all_stream_layers=all_stream,+ stream_layers=stream_layers,+ pg_state=pg_state, pg_stager=pg_stager)+ return model, model_config, shards, installs+++class Qwen35Engine(Edge0Engine):+ """Streaming Qwen3.5-MoE engine (staged decode + trained prerouter)."""++ name = "edge0-35b"++ def _build(self):+ cfg = self.cfg+ (self.model, self.model_config, self.shards,+ inst) = load_installed(cfg.model_dir, cfg)+ self._all_stream_layers = inst["all_stream_layers"]+ self._stream_layers = inst["stream_layers"]+ self._pg_state = inst["pg_state"]+ self._pg_stager = inst["pg_stager"]+ self._lm = self.model.language_model+ opts = cfg.options+ if self._tok is None:+ try:+ self._tok = load_tokenizer(cfg.model_dir)+ except Exception: # noqa: BLE001 — tokenizer optional for CLI+ pass++ self._prefill_before_layer = make_prefill_before_layer(+ self._all_stream_layers,+ full_n=getattr(opts, "prefill_full_layers", 0),+ hot_n=opts.prefill_hot, hot_window=cfg.hot_window)+ self._intra_after_layer = make_intra_after_layer(+ self._all_stream_layers, enabled=cfg.intra_staging)+ self._history_prefetch = make_history_prefetch(+ self._all_stream_layers, enabled=cfg.prefetch_history)++ self.cache = self._lm.make_cache()++ # ---- forward ----------------------------------------------------------++ def _forward(self, ids, intra_stage: bool = True) -> core.array:+ inputs = core.array(ids)[None, :]+ opts = self.cfg.options+ prefill_multi = len(ids) > 1 and self._prefill_active+ full_layer = bool(+ prefill_multi and self._all_stream_layers+ and opts.full_layer_prefill)+ before_cb = self._prefill_before_layer if prefill_multi else None+ if full_layer:+ after_cb = None+ else:+ after_cb = (self._intra_after_layer+ if (intra_stage and self._intra_after_layer is not None+ and len(ids) == 1)+ else None)+ h = self._lm.model(+ inputs, cache=self.cache, before_layer_cb=before_cb,+ after_layer_cb=after_cb,+ async_eval_per_layer=bool(prefill_multi and full_layer))+ logits = self._lm.lm_head(h[0, -1])+ core.eval(logits)+ # Step boundary: ONE stacked head batch -> ONE tolist -> fills.+ if (self._pg_stager is not None and not self._prefill_active+ and len(ids) == 1):+ self._pg_stager.stage_all()+ self._pg_state.swap()+ return logits++ # ---- step hooks -------------------------------------------------------++ def _step_pre(self, token_id: int) -> None:+ if self._pg_stager is not None:+ # Router layers (no owner): fill from their previous actuals.+ st = self._pg_state+ for li, exp in self._all_stream_layers.items():+ if li < st.start or li >= st.n - 1:+ if exp.last_used:+ exp.stage_experts(list(exp.last_used))+ elif self._history_prefetch is not None:+ self._history_prefetch()++ def _step_post(self, token_id: int) -> None:+ if self._pg_stager is not None:+ st = self._pg_state+ for li, exp in self._all_stream_layers.items():+ if li < st.start or li >= st.n - 1:+ exp.sync_actuals()+ return+ if self._stream_layers:+ for exp in self._all_stream_layers.values():+ exp.sync_actuals()+ exp.stage_experts(list(exp.last_used))+ exp.swap_staged()++ # ---- prefill / reset --------------------------------------------------++ def _prefill_end(self) -> None:+ for exp in self._all_stream_layers.values():+ exp.clear_full_layer()+ for exp in self._all_stream_layers.values():+ exp.refresh_hot_pins()+ if self._stream_layers:+ for exp in self._stream_layers.values():+ exp.stage_from_prefill()++ def _reset_state(self) -> None:+ self.cache = self._lm.make_cache()+ if self._pg_state is not None:+ self._pg_state.reset()++ def _lm_logits(self, h: core.array) -> core.array:+ return self._lm.lm_head(h[0, -1])diff --git a/src/edge0/models/__init__.py b/src/edge0/models/__init__.pynew file mode 100644index 0000000..9804d40--- /dev/null+++ b/src/edge0/models/__init__.py@@ -0,0 +1,17 @@+"""Model adapters: one package per tier, registered on import.++``edge0.models.base`` carries the shared ``ModelConfig``; the tier+packages (``edge0.models.edge0_35b``, ``edge0.models.edge0_10b``)+define their family configs and the ``build_model`` / ``build_engine``+entry points, then call ``register_model`` so ``AutoConfig`` /+``AutoModel`` / ``AutoEngine`` can resolve them by name.+"""++from __future__ import annotations++from edge0.models.base import ModelConfig++# Importing the tier packages populates edge0.registry.MODEL_REGISTRY.+from edge0.models import edge0_10b, edge0_35b # noqa: F401++__all__ = ["ModelConfig", "edge0_10b", "edge0_35b"]diff --git a/src/edge0/models/base.py b/src/edge0/models/base.pynew file mode 100644index 0000000..ad3bac8--- /dev/null+++ b/src/edge0/models/base.py@@ -0,0 +1,96 @@+"""Model adapters: per-model configs, profiles and build entry points.++The registry contract (``edge0.registry``) is: an adapter module exposes+``Config`` (a ``ModelConfig`` subclass with ``from_pretrained``),+``build_model`` and ``build_engine``. ``Config`` carries everything the+engines need — MoE spec, streaming options, prerouter spec, LoRA, sampling+defaults, server port and the acceptance profile (tokens/s + peak-active+memory targets measured on the production machine).+"""++from __future__ import annotations++import os+from dataclasses import dataclass, field, fields, replace+from pathlib import Path++from edge0.config import GenerationConfig+from edge0.moe.spec import MoESpec+from edge0.prerouter.spec import PrerouterSpec+from edge0.streaming.options import LayerOptions++# Repository-root artifacts (gitignored; produced once by+# scripts/convert_adapters_legacy.py from the training npz exports, then+# the source npz are discarded).+ARTIFACTS_DIR = Path(__file__).resolve().parents[3] / "artifacts"+++def artifact(name: str) -> str:+ """Absolute path of one converted adapter file under ``artifacts/``."""+ return str(ARTIFACTS_DIR / name)+++@dataclass+class ModelConfig:+ """Everything one edge0 model tier needs to run.++ Subclasses fix the family defaults (specs, presets, generation, port,+ acceptance targets); ``from_pretrained`` merges user overrides.+ """++ name: str+ model_dir: str+ moe_spec: MoESpec+ options: LayerOptions+ prerouter: PrerouterSpec | None = None+ prerouter_top_k: int = 0 # 0 -> options.top_k+ lora: str = "" # safetensors path; "" disables+ lora_r: int = 16+ lora_alpha: float = 32.0+ gen: GenerationConfig = field(default_factory=GenerationConfig)+ prefill_chunk: int = 2048+ hot_window: int = 4+ intra_staging: bool = False+ prefetch_history: bool = True+ port: int = 8000+ # acceptance profile (measured on the production Mac)+ target_tok_s: float = 0.0+ peak_active_mem_mb: float = 0.0++ @classmethod+ def from_pretrained(cls, model_dir: str | None = None, **overrides):+ """Build the family config, resolving artifact paths and applying+ per-field overrides (any public attribute may be overridden)."""+ if model_dir is None:+ model_dir = overrides.pop("model_dir", None) or ""+ base = cls._defaults(model_dir)+ for key, value in overrides.items():+ if not hasattr(base, key):+ raise TypeError(+ f"unknown {cls.__name__} override {key!r} "+ f"(known: {sorted(f.name for f in fields(base))})")+ base = replace(base, **{key: value})+ return base++ @classmethod+ def _defaults(cls, model_dir: str) -> "ModelConfig":+ raise NotImplementedError+++def resolve_prerouter(pspec: PrerouterSpec, weights_file: str) -> PrerouterSpec:+ """Fill the prerouter weights path (specs are frozen; only used when+ the caller didn't already set one)."""+ if pspec.weights_file:+ return pspec+ return replace(pspec, weights_file=weights_file)+++def resolve_lora(lora: str, model_dir: str, r: int, alpha: float):+ """Sanitize the LoRA override: a bare model name resolves to the+ artifact of the same tier, ``model_dir`` means "keep the training+ weights in place", empty string disables."""+ if lora == "model_dir":+ return (os.path.join(model_dir, "lora.safetensors")+ if os.path.isfile(os.path.join(model_dir, "lora.safetensors"))+ else "")+ return loradiff --git a/src/edge0/models/edge0_10b/__init__.py b/src/edge0/models/edge0_10b/__init__.pynew file mode 100644index 0000000..155bf24--- /dev/null+++ b/src/edge0/models/edge0_10b/__init__.py@@ -0,0 +1,100 @@+"""edge0-10b adapter: Ling 3.0 hybrid (MLA + MoE, 10B tier).++Family facts (production profile, verified against the v7 checkpoint):++* 24 layers (layer 0 dense), 128 routed experts (4-bit affine,+ group 64), K=8 native, sigmoid + group-limited routing+ (n_group 8, topk_group 4, routed scaling 2.5, norm_topk_prob),+ 1 resident shared expert, separate gate/up/down stacked tensors+ under ``model.layers.N.mlp.experts``.+* hybrid prerouter baked into the vendored model: 22 heads+ (explicit owners 1..22), start_layer 1, hidden 512, fp16,+ feature_topk "executed", consumed inside ``BailingSparseMoE``+ from ``prerouter_cache`` logits (patch_call=False).+* serving: port 8083; acceptance ≈33 tok/s, peak active ≈1.4 GB.+"""++from __future__ import annotations++import sys++from edge0.config import GenerationConfig+from edge0.models.base import ModelConfig, artifact+from edge0.moe.spec import MoESpec, QuantSpec, RouterKind, WeightLayout+from edge0.prerouter.spec import PrerouterSpec+from edge0.registry import register_model+from edge0.streaming.options import LayerOptions+++class Ling10BConfig(ModelConfig):+ """edge0-10b family config (Ling 3.0 hybrid, K=8)."""++ @classmethod+ def _defaults(cls, model_dir: str) -> "Ling10BConfig":+ return cls(+ name="edge0-10b",+ model_dir=model_dir,+ moe_spec=MoESpec(+ num_experts=128, top_k=8, intermediate_size=512,+ router=RouterKind.SIGMOID_GROUP, norm_topk_prob=True,+ routed_scaling=2.5, n_group=8, topk_group=4,+ shared_experts=1,+ quant=QuantSpec(bits=4, group_size=64, mode="affine"),+ layout=WeightLayout.SEPARATE,+ key_template="model.layers.{layer}.mlp.experts",+ block_path="model.layers.{layer}.mlp",+ layer_path="model.layers.{layer}",+ ),+ options=LayerOptions.prod_k8(),+ prerouter=PrerouterSpec(+ kind=RouterKind.SIGMOID_GROUP,+ start_layer=7, hidden=512, dtype="fp16",+ feature_topk="executed",+ owners=tuple(range(6, 23)),+ weights_file=artifact("prerouter_edge0_10b.safetensors"),+ patch_call=False,+ ),+ prerouter_top_k=8,+ lora=artifact("lora_edge0_10b.safetensors"),+ lora_r=16, lora_alpha=32.0,+ gen=GenerationConfig(+ temperature=0.7, top_p=0.95, top_k=64,+ repetition_penalty=1.1, max_new_tokens=2048,+ eos_ids=(156895,)),+ prefill_chunk=2048,+ hot_window=1,+ intra_staging=False,+ prefetch_history=True,+ port=8083,+ target_tok_s=33.0,+ peak_active_mem_mb=1400.0,+ )+++def build_model(model_dir: str | None = None, **overrides):+ """Load the ling skeleton with streaming twins / LoRA / prerouter+ installed (the engine's own build path, DRY-shared)."""+ from edge0.engine.ling import load_installed++ cfg = Ling10BConfig.from_pretrained(model_dir, **overrides)+ model, _mcfg, _shards, _installs = load_installed(cfg.model_dir, cfg)+ return model+++def build_engine(model_dir: str | None = None, **overrides):+ """Build a ready-to-generate Ling10BEngine.++ ``think`` (THINK_MODE parity, default False) selects the+ chat-template thinking mode for ``encode_chat``.+ """+ from edge0.engine.ling import Ling10BEngine++ think = overrides.pop("think", False)+ cfg = Ling10BConfig.from_pretrained(model_dir, **overrides)+ return Ling10BEngine(cfg.model_dir, cfg, think=think)+++register_model("edge0-10b", sys.modules[__name__])+++Config = Ling10BConfig # registry contract: adapter.Configdiff --git a/src/edge0/models/edge0_35b/__init__.py b/src/edge0/models/edge0_35b/__init__.pynew file mode 100644index 0000000..90805a4--- /dev/null+++ b/src/edge0/models/edge0_35b/__init__.py@@ -0,0 +1,93 @@+"""edge0-35b adapter: Qwen3.5-MoE (K=4 tier, 35B-class).++Family facts (production profile, verified against the round-7+checkpoint):++* 40 layers, 256 routed experts (4-bit affine, group 64), K=4,+ softmax->top-k routing with norm_topk_prob, 1 resident shared+ expert, separate gate/up/down stacked tensors under+ ``language_model.model.layers.N.mlp.switch_mlp``.+* trained prerouter: 33 heads (owners 6..38), start_layer 7,+ hidden 512, fp16, consumed directly by the patched MoE block at+ decode (patch_call=True, cross-token staged decode, K=4).+* serving: port 8085; acceptance ≈13 tok/s, peak active ≈3.3 GB.+"""++from __future__ import annotations++import sys++from edge0.config import GenerationConfig+from edge0.models.base import ModelConfig, artifact+from edge0.moe.spec import MoESpec, QuantSpec, RouterKind, WeightLayout+from edge0.prerouter.spec import PrerouterSpec+from edge0.registry import register_model+from edge0.streaming.options import LayerOptions+++class Qwen35Config(ModelConfig):+ """edge0-35b family config (Qwen3.5-MoE, K=4)."""++ @classmethod+ def _defaults(cls, model_dir: str) -> "Qwen35Config":+ return cls(+ name="edge0-35b",+ model_dir=model_dir,+ moe_spec=MoESpec(+ num_experts=256, top_k=4, intermediate_size=512,+ router=RouterKind.SOFTMAX_TOPK, norm_topk_prob=True,+ shared_experts=1,+ quant=QuantSpec(bits=4, group_size=64, mode="affine"),+ layout=WeightLayout.SEPARATE,+ key_template="language_model.model.layers.{layer}."+ "mlp.switch_mlp",+ block_path="language_model.model.layers.{layer}.mlp",+ layer_path="language_model.model.layers.{layer}",+ ),+ options=LayerOptions.staged_k4(),+ prerouter=PrerouterSpec(+ kind=RouterKind.SOFTMAX_TOPK,+ start_layer=7, hidden=512, dtype="fp16",+ feature_topk="executed",+ weights_file=artifact("prerouter_edge0_35b_k4.safetensors"),+ patch_call=True,+ ),+ prerouter_top_k=4,+ lora=artifact("lora_edge0_35b_k4.safetensors"),+ lora_r=16, lora_alpha=32.0,+ gen=GenerationConfig(+ temperature=0.6, top_p=0.95, top_k=64,+ repetition_penalty=1.0, max_new_tokens=2048,+ eos_ids=(248046, 248044)),+ prefill_chunk=2048,+ hot_window=4,+ intra_staging=False,+ prefetch_history=True,+ port=8085,+ target_tok_s=13.0,+ peak_active_mem_mb=3400.0,+ )+++def build_model(model_dir: str | None = None, **overrides):+ """Load the qwen skeleton with streaming twins / LoRA / prerouter+ installed (the engine's own build path, DRY-shared)."""+ from edge0.engine.qwen import load_installed++ cfg = Qwen35Config.from_pretrained(model_dir, **overrides)+ model, _mcfg, _shards, _installs = load_installed(cfg.model_dir, cfg)+ return model+++def build_engine(model_dir: str | None = None, **overrides):+ """Build a ready-to-generate Qwen35Engine."""+ from edge0.engine.qwen import Qwen35Engine++ cfg = Qwen35Config.from_pretrained(model_dir, **overrides)+ return Qwen35Engine(cfg.model_dir, cfg)+++register_model("edge0-35b", sys.modules[__name__])+++Config = Qwen35Config # registry contract: adapter.Configdiff --git a/src/edge0/moe/__init__.py b/src/edge0/moe/__init__.pynew file mode 100644index 0000000..70d1301--- /dev/null+++ b/src/edge0/moe/__init__.py@@ -0,0 +1,18 @@+"""MoE specs and routing math (shared by resident and streaming paths)."""++from edge0.moe.routing import group_select_from_logits, select_from_logits+from edge0.moe.spec import (+ MoESpec,+ QuantSpec,+ RouterKind,+ WeightLayout,+)++__all__ = [+ "MoESpec",+ "QuantSpec",+ "RouterKind",+ "WeightLayout",+ "select_from_logits",+ "group_select_from_logits",+]diff --git a/src/edge0/moe/routing.py b/src/edge0/moe/routing.pynew file mode 100644index 0000000..0a9d2e4--- /dev/null+++ b/src/edge0/moe/routing.py@@ -0,0 +1,70 @@+"""Routing math, shared by resident and streaming MoE paths.++These functions must stay bit-identical to the vendored base+implementations (they are extracted from them verbatim); parity tests pin+this.+"""++from __future__ import annotations++from edge0.backends import core+++def select_from_logits(logits, top_k: int, norm: bool = True):+ """Softmax-topk router: precise softmax -> top-k -> renormalize.++ Mirrors the Qwen3.5-MoE HF router semantics (norm_topk_prob=True).+ Returns ``(inds [..., k], scores [..., k])``.+ """+ gates = core.softmax(logits, axis=-1, precise=True)+ inds = core.argpartition(gates, kth=-top_k, axis=-1)[..., -top_k:]+ scores = core.take_along_axis(gates, inds, axis=-1)+ if norm:+ scores = scores / scores.sum(axis=-1, keepdims=True)+ return inds, scores+++def group_select_from_logits(logits, top_k: int, n_group: int,+ topk_group: int, routed_scaling: float,+ norm: bool = True, expert_bias=None):+ """Sigmoid + group-limited top-k router (DeepSeek-V3 / Bailing law).++ Selection scores are ``sigmoid(logits) + expert_bias``; the+ ``topk_group`` best groups (by sum of each group's top-2 selection+ scores) survive; the top-k experts within surviving groups are chosen+ by selection score; the WEIGHTS are the raw sigmoid scores of the+ chosen experts (bias excluded), normalized if ``norm`` and scaled by+ ``routed_scaling``.++ Returns ``(inds [..., k], scores [..., k])``.+ """+ scores = core.sigmoid(logits.astype(core.float32))+ select = scores+ if expert_bias is not None:+ select = scores + expert_bias++ b_shape = select.shape[:-1]+ k_drop = n_group - topk_group+ if k_drop > 0:+ grouped = select.reshape(+ *b_shape, n_group, select.shape[-1] // n_group)+ # Group score = sum of the top-2 selection scores in the group.+ top2 = core.topk(grouped, 2, axis=-1)+ group_scores = top2.sum(axis=-1)+ drop = core.argpartition(group_scores, kth=k_drop - 1,+ axis=-1)[..., :k_drop]+ masked = core.put_along_axis(+ grouped,+ core.expand_dims(drop, -1),+ core.array(-float("inf"), grouped.dtype),+ axis=-2,+ )+ select = masked.reshape(*b_shape, select.shape[-1])+ # topk_group == n_group keeps every group — nothing to drop.++ idx = core.argpartition(-select, kth=top_k - 1, axis=-1)[..., :top_k]+ w = core.take_along_axis(scores, idx, axis=-1)+ if norm:+ w = w / (w.sum(axis=-1, keepdims=True) + 1e-20)+ w = w * routed_scaling+ return idx, wdiff --git a/src/edge0/moe/spec.py b/src/edge0/moe/spec.pynew file mode 100644index 0000000..176c43c--- /dev/null+++ b/src/edge0/moe/spec.py@@ -0,0 +1,147 @@+"""MoE abstraction: routing kinds, quantized weight layouts, and the spec+that drives both the resident math and the streaming layer.++Every MoE block in a supported model is described by one ``MoESpec``; the+streaming subsystem consumes the spec (layout + key template) so one+generic ``StreamingSwitchGLU`` serves all models.+"""++from __future__ import annotations++from dataclasses import dataclass, field+from enum import Enum+from typing import Literal+++class RouterKind(Enum):+ """Router math families supported by edge0."""++ SOFTMAX_TOPK = "softmax_topk"+ """precise softmax -> top-k -> renormalize (Qwen3.5-MoE style,+ norm_topk_prob=True)."""++ SIGMOID_GROUP = "sigmoid_group"+ """sigmoid scores + group-limited top-k + routed scaling+ (DeepSeek-V3 / Bailing style)."""+++class WeightLayout(Enum):+ """How expert projections are stored in the checkpoint."""++ SEPARATE = "separate"+ """gate_proj / up_proj / down_proj as three stacked tensors."""++ FUSED_GATE_UP = "fused_gate_up"+ """gate_proj+up_proj fused into one stacked tensor (rows doubled+ along the out axis); down_proj separate."""+++@dataclass(frozen=True)+class QuantSpec:+ """Weight quantization description (per-tensor defaults)."""++ bits: int = 4+ group_size: int = 64+ mode: str = "affine"+++@dataclass(frozen=True)+class MoESpec:+ """Everything the framework needs to know about a model's MoE blocks.++ Attributes:+ num_experts: routed experts per layer.+ top_k: experts selected per token (routing width).+ intermediate_size: expert FFN hidden size (documentation only;+ the math derives sizes from the weight tensors).+ router: routing math family.+ norm_topk_prob: renormalize selection weights.+ routed_scaling: multiplier applied to selection weights+ (sigmoid-group routers); None for softmax-topk.+ n_group / topk_group: group-limited routing parameters+ (sigmoid-group routers only).+ shared_experts: number of resident shared experts (never streamed).+ quant: weight quantization.+ layout: weight layout.+ key_template: safetensors key prefix for layer ``{layer}``, with+ ``{proj}`` / ``{part}`` placeholders appended by the consumer,+ e.g. ``"language_model.model.layers.{layer}.mlp.switch_mlp"``.+ block_path: dotted attribute path from the loaded model object to+ the MoE block, with ``{layer}``, e.g.+ ``"language_model.model.layers.{layer}.mlp.switch_mlp"``.+ layer_path: dotted attribute path to the DECODER LAYER object+ (the block's owner), with ``{layer}``, e.g.+ ``"language_model.model.layers.{layer}"``. Used by prerouter+ stagers to read per-layer caches; default derived from+ ``block_path`` by stripping the trailing attribute parts.+ expert_row_axis: axis of the stacked [num_experts, ...] tensors+ along which one expert is a contiguous slice (always 0 for the+ supported layouts; kept for documentation).+ """++ num_experts: int+ top_k: int+ intermediate_size: int+ router: RouterKind = RouterKind.SOFTMAX_TOPK+ norm_topk_prob: bool = True+ routed_scaling: float | None = None+ n_group: int | None = None+ topk_group: int | None = None+ shared_experts: int = 0+ quant: QuantSpec = field(default_factory=QuantSpec)+ layout: WeightLayout = WeightLayout.SEPARATE+ key_template: str = ""+ block_path: str = ""+ layer_path: str = ""+ expert_row_axis: int = 0++ def keys(self, layer: int, proj: str, part: str) -> str:+ """Resolve one safetensors key for a layer's projection part."""+ prefix = self.key_template.format(layer=layer)+ return f"{prefix}.{proj}.{part}"++ def block_of(self, model, layer: int):+ """Resolve the MoE block object inside a loaded model."""+ obj = model+ for part in self.block_path.format(layer=layer).split("."):+ if part.isdigit():+ obj = obj[int(part)] # layers are plain lists in most families+ else:+ obj = getattr(obj, part)+ return obj++ def layer_of(self, model, layer: int):+ """Resolve the decoder layer object (the block's owner).++ Falls back to ``block_path`` minus the trailing attribute+ segments (the convention: a block lives at+ ``<layer>.<mlp>.<block>``, so the layer is two segments up).+ """+ if self.layer_path:+ path = self.layer_path.format(layer=layer)+ else:+ path = ".".join(+ self.block_path.format(layer=layer).split(".")[:-2])+ obj = model+ for part in path.split("."):+ if part.isdigit():+ obj = obj[int(part)]+ else:+ obj = getattr(obj, part)+ return obj++ @property+ def bundle_projs(self) -> tuple[str, ...]:+ """Projections a cached expert bundle contains, in stack order.++ Order matches the math signature directly: up, gate, down+ (``_swiglu(up, gate)``); fused layout packs gate rows on top of+ up rows so ``split(x_gu, 2)`` yields ``(gate, up)``.+ """+ if self.layout is WeightLayout.FUSED_GATE_UP:+ return ("gate_up_proj", "down_proj")+ return ("up_proj", "gate_proj", "down_proj")++ @property+ def fuse_gu(self) -> bool:+ return self.layout is WeightLayout.FUSED_GATE_UPdiff --git a/src/edge0/prerouter/__init__.py b/src/edge0/prerouter/__init__.pynew file mode 100644index 0000000..a646b23--- /dev/null+++ b/src/edge0/prerouter/__init__.py@@ -0,0 +1,31 @@+"""Prerouter: trained cross-token routing predictor heads + step-boundary+staging.++Public API::++ from edge0.prerouter import (PrerouterSpec, PrerouterState,+ PrerouterHead, install_prerouter,+ CrossTokenStager, LingPrerouterStager)+"""++from edge0.prerouter.heads import PrerouterHead, select_from_logits, topk_onehot+from edge0.prerouter.install import install_prerouter+from edge0.prerouter.spec import PrerouterSpec+from edge0.prerouter.stager import (+ CrossTokenStager,+ LingPrerouterStager,+ PrerouterStager,+)+from edge0.prerouter.state import PrerouterState++__all__ = [+ "PrerouterSpec",+ "PrerouterState",+ "PrerouterHead",+ "PrerouterStager",+ "CrossTokenStager",+ "LingPrerouterStager",+ "install_prerouter",+ "select_from_logits",+ "topk_onehot",+]diff --git a/src/edge0/prerouter/heads.py b/src/edge0/prerouter/heads.pynew file mode 100644index 0000000..a08867c--- /dev/null+++ b/src/edge0/prerouter/heads.py@@ -0,0 +1,73 @@+"""Prerouter head math — faithful port of the qwen trained-head+implementation (the legacy ``v7_trained`` file) with identifiers+renamed.++Semantics (cross-token):+ * features = concat[hidden, this-token executed top-k one-hot,+ previous-token executed top-k one-hot];+ * head = fc1(features) -> exact(erf) gelu -> fc2 + linear_init on the+ SAME concat features;+ * fp16 weights by default (the training export precision, and the+ precision ling v7 production ran the heads at); fp32 for debugging.+"""++from __future__ import annotations++import math++from edge0.backends import core+from edge0.backends import nn++_EYE_CACHE: dict[int, core.array] = {}+++def _eye(num_experts: int) -> core.array:+ e = _EYE_CACHE.get(num_experts)+ if e is None:+ e = core.eye(num_experts)+ _EYE_CACHE[num_experts] = e+ return e+++def topk_onehot(inds: core.array, num_experts: int) -> core.array:+ """inds [B,T,k] -> one-hot [B,T,num_experts]."""+ idm = _eye(num_experts)+ oh = core.take(idm, inds, axis=0) # [B,T,k,E]+ return oh.sum(axis=-2)+++def gelu_erf(x: core.array) -> core.array:+ """Exact GELU (erf) matching torch ``F.gelu`` default."""+ return 0.5 * x * (1.0 + core.erf(x / math.sqrt(2.0)))+++def select_from_logits(logits: core.array, top_k: int):+ """Mirror the qwen HF router math: precise softmax -> top-k ->+ renormalize. Returns (inds [..., k], scores [..., k])."""+ gates = core.softmax(logits, axis=-1, precise=True)+ inds = core.argpartition(gates, kth=-top_k, axis=-1)[..., -top_k:]+ scores = core.take_along_axis(gates, inds, axis=-1)+ scores = scores / scores.sum(axis=-1, keepdims=True)+ return inds, scores+++class PrerouterHead(nn.Module):+ """Trained prerouter head (fp16 by default; fp32 via ``dtype``)."""++ def __init__(self, hidden: int, num_experts: int, prerouter_hidden: int,+ dtype=core.float16):+ super().__init__()+ self._dtype = dtype+ f_in = hidden + 2 * num_experts+ self.fc1 = nn.Linear(f_in, prerouter_hidden, bias=False)+ self.fc2 = nn.Linear(prerouter_hidden, num_experts, bias=False)+ self.linear_init = nn.Linear(f_in, num_experts, bias=False)++ def __call__(self, h: core.array, executed_oh: core.array,+ prev_oh: core.array) -> core.array:+ feats = core.concatenate([h, executed_oh, prev_oh], axis=-1)+ if feats.dtype != self._dtype:+ feats = feats.astype(self._dtype)+ # linear_init consumes the full concat features, same as training.+ return self.linear_init(feats) + self.fc2(+ gelu_erf(self.fc1(feats)))diff --git a/src/edge0/prerouter/install.py b/src/edge0/prerouter/install.pynew file mode 100644index 0000000..13fa101--- /dev/null+++ b/src/edge0/prerouter/install.py@@ -0,0 +1,188 @@+"""Install prerouter heads into a loaded base model.++Two conventions exist (both supported here):++* qwen (``patch_call=True``): heads are NEW ``PrerouterHead`` modules+ attached to the MoE blocks, and the MoE block's ``__call__`` is patched+ at class level so decode routes through the prerouter's+ ``pred_inds``/``pred_scores`` (100% replacement, no matching/drops).+ Prefill is untouched (router path). The patch mirrors the deployment's+ proven ``_trained_call``, with guards per instance so non-prerouter+ engines keep the normal router.+* ling (``patch_call=False``): the vendored model already contains one+ ``BailingPrerouter`` per MoE layer and consumes ``prerouter_cache``+ logits internally; this installer only fills the head weights (and+ zeroes ``linear_init`` when the checkpoint has none).+"""++from __future__ import annotations++from edge0.backends import core+from edge0.backends import nn++from edge0.backends.mlx._impl.qwen3_next import Qwen3NextSparseMoeBlock+from edge0.backends.mlx.io import load_safetensors+from edge0.moe.spec import MoESpec+from edge0.prerouter.heads import PrerouterHead, topk_onehot+from edge0.prerouter.spec import PrerouterSpec+from edge0.prerouter.state import PrerouterState++_PATCHED_MARK = "_edge0_prerouter_patched"+++def _parse_weights(weights: dict[str, core.array], dtype):+ """``layers.<N>.<part>.weight`` keys -> ``{N: {"<part>.weight": mx}}``.++ Accepts both ``layers.N.fc1.weight`` and the nested+ ``layers.N.mlp.prerouter.fc1.weight`` form."""+ heads: dict[int, dict[str, core.array]] = {}+ for key, arr in weights.items():+ parts = key.split(".")+ if len(parts) < 4 or parts[0] != "layers":+ continue+ try:+ owner = int(parts[1])+ except ValueError:+ continue+ if parts[-2] not in ("fc1", "fc2", "linear_init"):+ continue+ if arr.dtype != dtype:+ arr = arr.astype(dtype)+ heads.setdefault(owner, {})[f"{parts[-2]}.weight"] = arr+ return heads+++def install_prerouter(+ *,+ model,+ spec: MoESpec,+ pspec: PrerouterSpec,+ weights: dict[str, core.array] | None = None,+ state: PrerouterState | None = None,+ n_layers: int,+) -> tuple[PrerouterState, dict[int, nn.Module]]:+ """Attach prerouter heads to a loaded model.++ Returns ``(state, heads)`` where ``state`` is the cross-token+ double-buffer the engine drives at the step boundary and ``heads``+ maps owner layer -> head module. ``weights`` is a flat+ ``name -> core.array`` dict (from ``load_safetensors``); when None the+ file from ``pspec.weights_file`` is loaded instead.+ """+ if weights is None:+ if not pspec.weights_file:+ raise ValueError(+ "install_prerouter needs weights or PrerouterSpec.weights_file")+ weights = load_safetensors(pspec.weights_file)+ dtype = core.float16 if pspec.dtype == "fp16" else core.float32+ head_weights = _parse_weights(weights, dtype)+ missing = [n for n in pspec.owner_layers(n_layers) if n not in head_weights]+ if missing:+ raise ValueError(+ f"prerouter weights missing owner layers {missing[:5]}... "+ f"(have {sorted(head_weights)})")++ if state is None:+ state = PrerouterState(n_layers, spec.num_experts,+ pspec.start_layer,+ owners=pspec.owner_layers(n_layers))++ heads: dict[int, nn.Module] = {}+ for owner in pspec.owner_layers(n_layers):+ block = spec.block_of(model, owner)+ w = head_weights[owner]+ if pspec.patch_call:+ # hidden = the MoE block's input width, read off the router's+ # gate projection (weight is [num_experts, dim]).+ dim = block.gate.weight.shape[1]+ head = PrerouterHead(dim, spec.num_experts, pspec.hidden,+ dtype=dtype)+ # hidden size: MoESpec doesn't carry it; read from the gate+ # projection weight shape.+ head.fc1.weight = w["fc1.weight"]+ head.fc2.weight = w["fc2.weight"]+ head.linear_init.weight = w["linear_init.weight"]+ block.prerouter_head = head+ block.prerouter_li = owner+ block.prerouter_start = pspec.start_layer+ block.prerouter_enabled = True+ block.prerouter_state = state+ heads[owner] = head+ else:+ pg = getattr(block, "prerouter", None)+ if pg is None:+ raise AttributeError(+ f"layer {owner}: patch_call=False but block has no "+ "'prerouter' head (wrong model family?)")+ pg.fc1 = nn.Linear(w["fc1.weight"].shape[1],+ w["fc1.weight"].shape[0], bias=False)+ pg.fc1.weight = w["fc1.weight"]+ pg.fc2 = nn.Linear(w["fc2.weight"].shape[1],+ w["fc2.weight"].shape[0], bias=False)+ pg.fc2.weight = w["fc2.weight"]+ wk = w.get("linear_init.weight")+ if wk is not None:+ pg.linear_init = nn.Linear(wk.shape[1], wk.shape[0],+ bias=False)+ pg.linear_init.weight = wk+ else:+ pg.linear_init.weight = core.zeros(+ pg.linear_init.weight.shape, dtype=dtype)+ heads[owner] = pg++ if pspec.patch_call and not getattr(+ Qwen3NextSparseMoeBlock, _PATCHED_MARK, False):+ _patch_qwen_consume()+ return state, heads+++def _patch_qwen_consume():+ """Patch ``Qwen3NextSparseMoeBlock.__call__`` (deploy-proven+ ``_trained_call`` semantics).++ A class-level patch is required because implicit ``moe(x)`` calls look+ up the type, not the instance. Guarded per instance: only blocks with+ ``prerouter_enabled`` and single-token inputs (decode) route through+ the prerouter; everything else takes the original router path."""+ orig_call = Qwen3NextSparseMoeBlock.__call__++ def prerouter_call(self, x):+ sm = getattr(self, "switch_mlp", None)+ st = getattr(self, "prerouter_state", None)+ if not (getattr(self, "prerouter_enabled", False) and st is not None+ and x.ndim == 3 and x.shape[1] == 1+ and getattr(sm, "_staged_replace", None) is False+ and sm is not None):+ return orig_call(self, x)+ li = self.prerouter_li+ # This layer CONSUMES the prediction produced by layer li-1's head+ # at the previous token (prev-layer + prev-token double shift).+ pred = st.pred_inds[li] if li >= st.start else None+ if pred is not None:+ inds = pred+ scores = st.pred_scores[li]+ else:+ # First decode tokens / router layers below start: fall back to+ # the real router (matches training's pos-0 fallback).+ gates = core.softmax(self.gate(x), axis=-1, precise=True)+ k = self.top_k+ inds = core.argpartition(gates, kth=-k, axis=-1)[..., -k:]+ scores = core.take_along_axis(gates, inds, axis=-1)+ if self.norm_topk_prob:+ scores = scores / scores.sum(axis=-1, keepdims=True)++ oh = topk_onehot(inds, self.num_experts)+ st.oh[li] = oh++ # Capture for the head this layer OWNS (runs at the step boundary).+ self.prerouter_m_in = x+ self.prerouter_oh = oh++ y = self.switch_mlp(x, inds)+ y = (y * scores[..., None]).sum(axis=-2)+ shared_y = self.shared_expert(x)+ shared_y = core.sigmoid(self.shared_expert_gate(x)) * shared_y+ return y + shared_y++ Qwen3NextSparseMoeBlock.__call__ = prerouter_call+ setattr(Qwen3NextSparseMoeBlock, _PATCHED_MARK, True)diff --git a/src/edge0/prerouter/spec.py b/src/edge0/prerouter/spec.pynew file mode 100644index 0000000..4789109--- /dev/null+++ b/src/edge0/prerouter/spec.py@@ -0,0 +1,65 @@+"""Prerouter abstraction: the trained cross-token routing predictor.++A prerouter head is OWNED by layer N: it runs on layer N's MoE-input+hidden state (post-attention norm output) at token t and predicts layer+N+1's routing. Layer N+1 consumes the prediction from layer N's head+at token t-1 (cross-token double shift: prev-layer + prev-token), so the+staged expert set for a decode step IS the prerouter's routing — zero+drops by construction.++Feature vector per head: ``concat[hidden, this-token top-k one-hot,+prev-token top-k one-hot]``; head = ``fc1 -> exact(erf) gelu -> fc2 ++linear_init`` on the same features.+"""++from __future__ import annotations++from dataclasses import dataclass, field++from edge0.moe.spec import RouterKind+++@dataclass(frozen=True)+class PrerouterSpec:+ """Configuration of one model's prerouter.++ Attributes:+ kind: routing math family the prerouter head's logits are turned+ into expert selections with. ``SOFTMAX_TOPK`` (qwen:+ precise softmax -> top-k -> renormalize) or+ ``SIGMOID_GROUP`` (ling: sigmoid + group-limited top-k, the+ model's own ``_select_from_logits``).+ start_layer: first CONSUMING layer; the first owner is+ ``start_layer - 1`` and the last owner ``n_layers - 2``.+ hidden: prerouter hidden width (512 for both shipped models).+ dtype: head compute dtype — "fp16" (default, training export+ precision) or "fp32" (debug only).+ feature_topk: which one-hot feeds the "this-token top-k" feature.+ "executed": the top-k the block actually routed (qwen+ trained semantics — executed == prerouter-selected at decode);+ "teacher": the original gate's top-k, recomputed per layer+ (ling v7 training semantics).+ owners: explicit owner list override (defaults to+ ``range(start_layer - 1, n_layers - 1)``).+ weights_file: safetensors path with per-owner head weights+ (keys ``layers.<N>.fc1.weight`` etc.); may be provided later+ via ``install_prerouter``.+ patch_call: for models whose MoE block is NOT prerouter-aware+ (qwen3_next), install a class-level ``__call__`` patch so+ decode routes through ``pred_inds``. Models with baked hooks+ (bailing_hybrid) set this False.+ """++ kind: RouterKind = RouterKind.SOFTMAX_TOPK+ start_layer: int = 7+ hidden: int = 512+ dtype: str = "fp16"+ feature_topk: str = "executed" # "executed" | "teacher"+ owners: tuple[int, ...] | None = None+ weights_file: str = ""+ patch_call: bool = False++ def owner_layers(self, n_layers: int) -> list[int]:+ if self.owners is not None:+ return list(self.owners)+ return list(range(self.start_layer - 1, n_layers - 1))diff --git a/src/edge0/prerouter/stager.py b/src/edge0/prerouter/stager.pynew file mode 100644index 0000000..2c8a613--- /dev/null+++ b/src/edge0/prerouter/stager.py@@ -0,0 +1,244 @@+"""Prerouter step-boundary stager.++After each forward's logits eval, run every owner head as ONE stacked+batch (the base graph is already materialized), turn the logits into+expert selections with ONE eval + ONE tolist, and submit the next+decode step's staged slots per consumer layer — so the pool's SSD+builds overlap the next forward entirely (no per-layer sync points+inside the graph).++Two feature/consume conventions exist across the supported models, both+implemented here:++* qwen (``CrossTokenStager``): heads installed on the MoE blocks, the+ block patch captures ``(m_in, executed one-hot)``; the stager makes+ the selection itself (precise softmax -> top-k -> renormalize) and+ stores ``pred_inds`` / ``pred_scores`` keyed by the consuming layer.+* ling (``LingPrerouterStager``): heads baked into the vendored model,+ features read from the model's own caches (``m_in_cache`` /+ ``last_topk`` teacher feature / ``prev_topk_oh``); the stager stores+ the raw LOGITS in ``pg_cache`` and the consuming MoE block performs+ the sigmoid group-limited selection internally.+"""++from __future__ import annotations++from edge0.backends import core++from edge0.moe.routing import group_select_from_logits, select_from_logits+from edge0.moe.spec import MoESpec, RouterKind+from edge0.prerouter.heads import topk_onehot+from edge0.prerouter.spec import PrerouterSpec+from edge0.prerouter.state import PrerouterState+from edge0.streaming.layer import StreamingSwitchGLU+++def _topk_onehot_pos(idx, num_experts: int, pos: int = -1):+ """One-hot set of the top-k experts at position ``pos`` of ``idx``+ ([B, T, K] -> [B, 1, E], OR over the K slots) — ling feature+ semantics. Returns None when ``idx`` has no such position (e.g. a+ T=1 prefill asking for pos=-2) so the caller can fall back to the+ zeros feature (the deployment's ``_topk_onehot`` crashes on this+ case; the T=1 prompt is one BPE token like 你好 = [34355])."""+ if idx is None or idx.ndim < 3:+ return None+ t = idx.shape[-2]+ lo = pos if pos >= 0 else t + pos+ if lo < 0 or lo >= t:+ return None+ idx = idx[..., pos, :][..., None, :] # [B, 1, K]+ ar = core.arange(num_experts, dtype=idx.dtype)+ oh = (idx[..., None] == ar) # [B, 1, K, E]+ return oh.sum(axis=-2).astype(core.float32)+++class PrerouterStager:+ """Base: one-batch collect -> stack -> select -> eval -> tolist ->+ per-consumer stage_experts."""++ def __init__(+ self,+ *,+ model,+ spec: MoESpec,+ pspec: PrerouterSpec,+ state: PrerouterState,+ stream_layers: dict[int, StreamingSwitchGLU],+ top_k: int,+ ):+ self.model = model+ self.spec = spec+ self.pspec = pspec+ self.state = state+ self.stream_layers = stream_layers+ self.top_k = top_k+ self.num_experts = spec.num_experts+ self.records: list | None = None+ self.cur_step = 0++ # ---- feature hooks (overridden per family) ----------------------------++ def _features(self, owner: int):+ """Return (m_in, cur_oh, prev_oh) for the owner layer's head."""+ raise NotImplementedError++ def _select(self, logits: core.array):+ """Turn stacked head logits into (inds, scores)."""+ raise NotImplementedError++ def _store(self, consumer: int, logits_i, inds_i, scores_i,+ experts: list[int]) -> None:+ """Record this consumer's prediction + submit its staged fill."""+ raise NotImplementedError++ def _note(self, owner: int, cur_oh) -> None:+ """Per-owner side effects (e.g. roll the prev-token feature)."""++ # ---- the step-boundary batch ------------------------------------------++ def stage_all(self) -> dict[int, list[int]]:+ metas: list[int] = []+ inputs: list[core.array] = []+ cur_ohs: list[core.array] = []+ prev_ohs: list[core.array] = []+ for owner in self.state.owners:+ feats = self._features(owner)+ if feats is None:+ continue+ m_in, cur_oh, prev_oh = feats+ metas.append(owner)+ inputs.append(m_in)+ cur_ohs.append(cur_oh)+ prev_ohs.append(prev_oh)+ if not metas:+ return {}+ per_logits = []+ for owner, m_in, co, po in zip(metas, inputs, cur_ohs, prev_ohs):+ head = self._head_of(owner)+ per_logits.append(head(m_in, co, po))+ logits = core.stack(per_logits, axis=0) # [n,1,1,E]+ inds_all, scores_all = self._select(logits)+ core.eval(inds_all, scores_all)+ flat = inds_all.reshape(len(metas), -1).tolist() # ONE tolist+ expert_sets: dict[int, list[int]] = {}+ for i, owner in enumerate(metas):+ consumer = owner + 1+ experts = sorted(set(int(v) for v in flat[i]))+ expert_sets[consumer] = experts+ self._store(consumer, logits[i], inds_all[i],+ scores_all[i], experts)+ self._note(owner, cur_ohs[i])+ if self.records is not None:+ for owner, consumer in zip(metas, metas):+ self.records.append(+ (self.cur_step, consumer, expert_sets[consumer]))+ self.cur_step += 1+ return expert_sets++ def _head_of(self, owner: int):+ raise NotImplementedError+++class CrossTokenStager(PrerouterStager):+ """qwen semantics: heads on the MoE blocks, selection made here,+ ``pred_inds`` / ``pred_scores`` keyed by the consuming layer."""++ def __init__(self, *, model, spec: MoESpec, pspec: PrerouterSpec,+ state: PrerouterState,+ stream_layers: dict[int, StreamingSwitchGLU], top_k: int):+ super().__init__(model=model, spec=spec, pspec=pspec,+ state=state, stream_layers=stream_layers,+ top_k=top_k)++ def _head_of(self, owner: int):+ block = self.spec.block_of(self.model, owner)+ return block.prerouter_head++ def _features(self, owner: int):+ block = self.spec.block_of(self.model, owner)+ m_in = getattr(block, "prerouter_m_in", None)+ oh = getattr(block, "prerouter_oh", None)+ if m_in is None or oh is None:+ return None+ prev_oh = self.state.oh_prev[owner]+ if prev_oh is None:+ prev_oh = core.zeros((1, 1, self.num_experts), dtype=oh.dtype)+ return m_in, oh, prev_oh++ def _select(self, logits: core.array):+ return select_from_logits(logits, self.top_k)++ def _store(self, consumer: int, logits_i, inds_i, scores_i,+ experts: list[int]):+ st = self.state+ st.pred_inds[consumer] = inds_i+ st.pred_scores[consumer] = scores_i+ st.logits[consumer - 1] = logits_i+ exp = self.stream_layers.get(consumer)+ if exp is not None:+ exp.stage_experts(experts)++ def _note(self, owner: int, cur_oh) -> None:+ pass+++class LingPrerouterStager(PrerouterStager):+ """ling semantics: heads baked into the vendored model, raw logits+ cached per consumer layer (the MoE block re-selects internally with+ its own ``_select_from_logits``); teacher top-8 feature; the+ prev-token feature is rolled on the block."""++ def __init__(self, *, model, spec: MoESpec, pspec: PrerouterSpec,+ state: PrerouterState,+ stream_layers: dict[int, StreamingSwitchGLU], top_k: int):+ super().__init__(model=model, spec=spec, pspec=pspec,+ state=state, stream_layers=stream_layers,+ top_k=top_k)+ self.pg_cache: dict[int, core.array] = {}++ def _head_of(self, owner: int):+ block = self.spec.block_of(self.model, owner)+ return block.prerouter++ def _features(self, owner: int):+ layer = self.spec.layer_of(self.model, owner)+ block = self.spec.block_of(self.model, owner)+ m = getattr(layer, "m_in_cache", None)+ if m is None:+ return None+ m_in = m[..., -1:, :]+ last_topk = getattr(block, "last_topk", None)+ cur_oh = _topk_onehot_pos(last_topk, self.num_experts, pos=-1)+ prev_oh = getattr(block, "prev_topk_oh", None)+ if prev_oh is None:+ # Initial staging right after prefill: the "prev token" feature+ # is the second-to-last prefill position.+ prev_oh = _topk_onehot_pos(last_topk, self.num_experts, pos=-2)+ if cur_oh is None:+ cur_oh = core.zeros((1, 1, self.num_experts), dtype=core.float32)+ if prev_oh is None:+ prev_oh = core.zeros((1, 1, self.num_experts), dtype=core.float32)+ return m_in, cur_oh, prev_oh++ def _select(self, logits: core.array):+ return group_select_from_logits(+ logits, self.top_k,+ n_group=self.spec.n_group, topk_group=self.spec.topk_group,+ routed_scaling=self.spec.routed_scaling or 1.0)++ def _store(self, consumer: int, logits_i, inds_i, scores_i,+ experts: list[int]):+ self.pg_cache[consumer] = logits_i+ exp = self.stream_layers.get(consumer)+ if exp is not None:+ exp.stage_experts(experts)++ def _note(self, owner: int, cur_oh) -> None:+ block = self.spec.block_of(self.model, owner)+ # Remember this token's top-k so the next decode step can feed the+ # "prev-token" one-hot feature.+ block.prev_topk_oh = cur_oh++ def reset(self):+ self.pg_cache = {}+ self.cur_step = 0diff --git a/src/edge0/prerouter/state.py b/src/edge0/prerouter/state.pynew file mode 100644index 0000000..c996314--- /dev/null+++ b/src/edge0/prerouter/state.py@@ -0,0 +1,42 @@+"""Prerouter decode-time state (cross-token, cross-layer double buffer).++After each decode step call ``swap()``:+ ``logits_prev[owner] = logits[owner]`` (owner = producing layer N)+ ``oh_prev[layer] = oh[layer]`` (per-layer executed one-hots)+``pred_inds`` / ``pred_scores`` are keyed by the CONSUMING layer (N+1)+and hold the selection made at the step boundary; each is consumed+exactly once on the next step by that layer, then overwritten.+"""++from __future__ import annotations+++class PrerouterState:+ def __init__(self, n_layers: int, num_experts: int, start_layer: int,+ owners=None):+ self.n = n_layers+ self.num_experts = num_experts+ self.start = start_layer # first consuming layer+ if owners is None:+ owners = list(range(start_layer - 1, n_layers - 1))+ self.owners = list(owners)+ self.logits: list = [None] * n_layers # owner output, this token+ self.logits_prev: list = [None] * n_layers # owner output, prev token+ self.pred_inds: list = [None] * n_layers # keyed by consumer N+1+ self.pred_scores: list = [None] * n_layers # keyed by consumer N+1+ self.oh: list = [None] * n_layers # executed one-hots, now+ self.oh_prev: list = [None] * n_layers # executed one-hots, prev++ def swap(self):+ self.logits_prev = self.logits+ self.logits = [None] * self.n+ self.oh_prev = self.oh+ self.oh = [None] * self.n++ def reset(self):+ self.logits = [None] * self.n+ self.logits_prev = [None] * self.n+ self.pred_inds = [None] * self.n+ self.pred_scores = [None] * self.n+ self.oh = [None] * self.n+ self.oh_prev = [None] * self.ndiff --git a/src/edge0/registry.py b/src/edge0/registry.pynew file mode 100644index 0000000..3827c40--- /dev/null+++ b/src/edge0/registry.py@@ -0,0 +1,111 @@+"""Transformers-style model registry: name -> adapter class.++Adapters live in ``edge0.models`` and register themselves on import;+``AutoConfig`` / ``AutoModel`` / ``AutoEngine`` resolve a model by+explicit name, by the checkpoint's ``config.json`` ``model_type`` (via+``TYPE_ALIASES``), or by the checkpoint directory basename. Unknown+names raise ``KeyError`` listing the registered models.+"""++from __future__ import annotations++from dataclasses import dataclass++MODEL_REGISTRY: dict[str, type] = {}+TYPE_ALIASES: dict[str, str] = {+ # qwen3_5_moe family+ "qwen3_5_moe_text": "edge0-35b",+ "qwen3_5_moe": "edge0-35b",+ # ling family (config.json carries architectures, not model_type)+ "bailing_hybrid": "edge0-10b",+ "bailing_moe_linear": "edge0-10b",+ "BailingMoeV3ForCausalLM": "edge0-10b",+}+++def register_model(name: str, adapter: type) -> None:+ if name in MODEL_REGISTRY:+ raise ValueError(f"model {name!r} already registered")+ MODEL_REGISTRY[name] = adapter+++@dataclass(frozen=True)+class _RegisteredModel:+ """One entry of the registry, for the KeyError listing."""++ name: str+ adapter: type++ def __str__(self) -> str: # pragma: no cover - cosmetic+ return f"{self.name} ({self.adapter.__module__}.{self.adapter.__name__})"+++def _resolve_name(model_dir: str | None, name: str | None) -> str:+ if name:+ key = name+ elif model_dir:+ key = _model_type_from_dir(model_dir)+ else:+ raise ValueError("AutoX.from_pretrained needs model_dir or name")+ if key in MODEL_REGISTRY:+ return key+ if key in TYPE_ALIASES:+ return TYPE_ALIASES[key]+ known = ", ".join(sorted(MODEL_REGISTRY))+ raise KeyError(+ f"unknown model {key!r} (resolved from name={name!r}, "+ f"model_dir={model_dir!r}); registered: {known}")++def _model_type_from_dir(model_dir: str) -> str:+ """config.json model_type (or architectures[0] for checkpoints that+ omit model_type, e.g. the ling family), falling back to the+ directory basename."""+ import json+ import os+ cfg_path = os.path.join(model_dir, "config.json")+ if os.path.isfile(cfg_path):+ try:+ with open(cfg_path) as f:+ cfg = json.load(f)+ mt = str(cfg.get("model_type", "") or "")+ if not mt:+ archs = cfg.get("architectures") or []+ if archs:+ mt = str(archs[0])+ if mt:+ return mt+ except Exception:+ pass+ base = os.path.basename(os.path.normpath(model_dir))+ return base if base not in ("", ".") else ""+++class AutoConfig:+ """Resolve a model's config class from the registry."""++ @classmethod+ def from_pretrained(cls, model_dir=None, name=None, **kwargs):+ from edge0 import models # noqa: F401 (populates MODEL_REGISTRY)+ key = _resolve_name(model_dir, name)+ return MODEL_REGISTRY[key].Config.from_pretrained(+ model_dir, **kwargs)+++class AutoModel:+ """Load a base model + install streaming experts / prerouter / LoRA."""++ @classmethod+ def from_pretrained(cls, model_dir=None, name=None, **kwargs):+ from edge0 import models # noqa: F401+ key = _resolve_name(model_dir, name)+ return MODEL_REGISTRY[key].build_model(model_dir, **kwargs)+++class AutoEngine:+ """Build a ready-to-generate engine for a model tier."""++ @classmethod+ def from_pretrained(cls, model_dir=None, name=None, **kwargs):+ from edge0 import models # noqa: F401+ key = _resolve_name(model_dir, name)+ return MODEL_REGISTRY[key].build_engine(model_dir, **kwargs)diff --git a/src/edge0/sampling.py b/src/edge0/sampling.pynew file mode 100644index 0000000..8d51aa7--- /dev/null+++ b/src/edge0/sampling.py@@ -0,0 +1,61 @@+"""Token sampling — port of the deployment's HF-style sampler+(vectorized repetition penalty; one host-sync categorical draw)."""++from __future__ import annotations++from edge0.backends import core+++def _mask_logits(logits, temperature, top_k, top_p):+ """Temperature / top-k / top-p truncation, shared by ``sample`` and+ any speculative verifier so both draw from the exact same+ distribution."""+ logits = logits.astype(core.float32)+ if temperature > 0:+ logits = logits / temperature+ if top_k is not None and top_k > 0:+ k = min(top_k, logits.shape[-1])+ top = core.topk(logits, k, axis=-1)+ # core.topk returns values in ASCENDING order, so the smallest+ # surviving value is the FIRST entry (top[..., 0]), not the last.+ threshold = top[..., :1]+ logits = core.where(logits < threshold, float("-inf"), logits)+ if top_p is not None and top_p < 1.0:+ sorted_vals = core.sort(logits, axis=-1)[..., ::-1]+ cum = core.cumsum(core.softmax(sorted_vals, axis=-1), axis=-1)+ cutoff = cum <= top_p+ counts = core.sum(cutoff.astype(core.int32), axis=-1)+ k = core.maximum(counts, 1)+ threshold = core.take_along_axis(+ sorted_vals, (k - 1)[..., None], axis=-1)+ logits = core.where(logits < threshold, float("-inf"), logits)+ if temperature <= 0:+ # Deterministic decode (HF / llama.cpp semantics): temperature+ # <= 0 means greedy — keep only the argmax (after any top-k+ # truncation above), so categorical always draws it.+ logits = core.where(+ logits < core.max(logits), float("-inf"), logits)+ return logits+++def sample(logits, temperature=0.7, top_k=None, top_p=None,+ repetition_penalty=1.0, history=(), seed=None) -> int:+ """Sample one token id from raw logits (HF-style penalty).++ ``history`` is an iterable of already-generated token ids; the+ penalty applies to them (vectorized, no per-token host syncs).+ """+ if repetition_penalty != 1.0 and history:+ logits = logits.astype(core.float32)+ ids = core.array(sorted(set(int(x) for x in history)),+ dtype=core.int32)+ if ids.size:+ vals = core.take(logits, ids)+ logits[ids] = core.where(+ vals > 0,+ vals / repetition_penalty,+ vals * repetition_penalty)+ logits = _mask_logits(logits, temperature, top_k, top_p)+ if seed is not None:+ core.random.seed(seed)+ return int(core.random.categorical(logits[None]).item())diff --git a/src/edge0/server/__init__.py b/src/edge0/server/__init__.pynew file mode 100644index 0000000..473606a--- /dev/null+++ b/src/edge0/server/__init__.py@@ -0,0 +1,12 @@+"""edge0 serving layer: OpenAI-compatible chat endpoint over one engine."""++from edge0.server.chat import (ChatMessage, ChatRequest, ChatSession,+ QueueServer, decode_tokens, parse_chat_request,+ sse_format)+from edge0.server.app import (create_app, run_server, run_stdlib)++__all__ = [+ "ChatMessage", "ChatRequest", "ChatSession", "QueueServer",+ "decode_tokens", "parse_chat_request", "sse_format",+ "create_app", "run_server", "run_stdlib",+]diff --git a/src/edge0/server/app.py b/src/edge0/server/app.pynew file mode 100644index 0000000..0de46a9--- /dev/null+++ b/src/edge0/server/app.py@@ -0,0 +1,226 @@+"""HTTP app layer (Flask optional; stdlib fallback).++Preference order: if ``flask`` is importable it is used (the full+OpenAI-compatible surface with SSE streaming); otherwise a minimal+stdlib ``http.server`` handler serves the same JSON contract+(non-streaming). Both share ``build_app_handlers`` so the route+semantics stay identical.+"""++from __future__ import annotations++import json+import threading+import time+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from urllib.parse import urlparse++from edge0.server.chat import (QueueServer, parse_chat_request, sse_format,+ decode_tokens)++try: # pragma: no cover - environment dependent+ from flask import Flask, Response, jsonify, request # type: ignore++ _HAS_FLASK = True+except ImportError: # pragma: no cover+ _HAS_FLASK = False+++def _error(status: int, msg: str) -> tuple:+ if _HAS_FLASK:+ return jsonify({"error": {"message": msg, "type": "invalid_request"}}), status+ return (json.dumps({"error": {"message": msg}}), status)+++def _split_think(text: str, think: bool):+ """ling-mlx-server server.py parity: with thinking enabled the model+ streams the reasoning block first and closes it with ``</think>``+ (token 156904); everything before is ``reasoning_content``, after+ is ``content``. A generation cut off inside the reasoning block+ surfaces what it has so the UI isn't a blank wait."""+ if think:+ if "</think>" in text:+ r, c = text.split("</think>", 1)+ return r.strip(), c.strip()+ return text.strip(), ""+ return "", text+++def _chat_once(server: QueueServer, payload: dict):+ req = parse_chat_request(payload)+ tokens, meta = server.chat(req)+ text = decode_tokens(server.engine, tokens)+ think = bool(req.enable_thinking if req.enable_thinking is not None+ else getattr(server.engine, "think", False))+ reasoning, content = _split_think(text, think)+ return {+ "id": f"chatcmpl-{int(time.time() * 1000)}",+ "object": "chat.completion",+ "created": int(time.time()),+ "model": server.model_name,+ "choices": [{+ "index": 0,+ "message": {+ "role": "assistant",+ "content": content,+ "reasoning_content": reasoning,+ },+ "finish_reason": "stop",+ }],+ "usage": meta["usage"],+ }+++def _chat_stream(server: QueueServer, payload: dict):+ req = parse_chat_request(payload)+ events: list[bytes] = []++ def on_token(tid: int):+ text = decode_tokens(server.engine, [tid])+ events.append(sse_format({+ "id": "chatcmpl-1", "object": "chat.completion.chunk",+ "created": int(time.time()), "model": server.model_name,+ "choices": [{"index": 0,+ "delta": {"content": text},+ "finish_reason": None}],+ }).encode("utf-8"))++ tokens, meta = server.chat(req, on_token=on_token)+ events.append(sse_format({+ "id": "chatcmpl-1", "object": "chat.completion.chunk",+ "created": int(time.time()), "model": server.model_name,+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],+ "usage": meta["usage"],+ }).encode("utf-8"))+ events.append(b"data: [DONE]\n\n")+ return b"".join(events)+++def build_app_handlers(server: QueueServer):+ """Return a handler dispatch dict shared by both transports."""++ def handle_chat(payload: dict):+ if payload.get("stream"):+ if _HAS_FLASK:+ return Response(_chat_stream(server, payload),+ mimetype="text/event-stream")+ return (_chat_stream(server, payload), 200,+ {"Content-Type": "text/event-stream"})+ return _chat_once(server, payload)++ handlers = {+ "GET /healthz": lambda: {"status": "ok", "model": server.model_name},+ "GET /v1/models": lambda: {+ "object": "list",+ "data": [{+ "id": server.model_name,+ "object": "model",+ "owned_by": "edge0",+ }],+ },+ "POST /v1/chat/completions": handle_chat,+ "POST /v1/completions": lambda p: _error(+ 400, "text completions not supported; use /v1/chat/completions"),+ }+ return handlers+++def create_app(server: QueueServer):+ """Flask app (raises ImportError when flask is missing)."""+ if not _HAS_FLASK:+ raise ImportError("flask is required for create_app(); "+ "use run_stdlib instead")+ app = Flask("edge0")+ handlers = build_app_handlers(server)++ @app.get("/healthz")+ def healthz():+ return handlers["GET /healthz"]()++ @app.get("/v1/models")+ def models():+ return handlers["GET /v1/models"]()++ @app.post("/v1/chat/completions")+ def chat():+ payload = request.get_json(force=True, silent=True) or {}+ out = handlers["POST /v1/chat/completions"](payload)+ if isinstance(out, Response):+ return out+ if isinstance(out, tuple) and out and isinstance(out[0], Response):+ return out+ return jsonify(out)++ @app.post("/v1/completions")+ def completions():+ payload = request.get_json(force=True, silent=True) or {}+ out = handlers["POST /v1/completions"](payload)+ if isinstance(out, tuple):+ return out+ return jsonify(out)++ return app+++class _StdlibHandler(BaseHTTPRequestHandler):+ server_q = None # type: QueueServer++ def _json(self, status: int, obj):+ body = json.dumps(obj, ensure_ascii=False).encode("utf-8")+ self.send_response(status)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def do_GET(self): # noqa: N802+ path = urlparse(self.path).path+ handlers = build_app_handlers(self.server_q)+ if path == "/healthz":+ self._json(200, handlers["GET /healthz"]())+ elif path == "/v1/models":+ self._json(200, handlers["GET /v1/models"]())+ else:+ self._json(404, {"error": {"message": f"no route {path}"}})++ def do_POST(self): # noqa: N802+ path = urlparse(self.path).path+ handlers = build_app_handlers(self.server_q)+ if path not in ("/v1/chat/completions", "/v1/completions"):+ self._json(404, {"error": {"message": f"no route {path}"}})+ return+ length = int(self.headers.get("Content-Length", 0))+ payload = json.loads(self.rfile.read(length) or b"{}")+ out = handlers[("POST " + path)](payload)+ if isinstance(out, tuple):+ body, status, headers = out+ self.send_response(status)+ for k, v in headers.items():+ self.send_header(k, v)+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)+ else:+ self._json(200, out)++ def log_message(self, fmt, *args): # quiet by default+ pass+++def run_stdlib(server: QueueServer, host: str, port: int):+ handler = type("Edge0Handler", (_StdlibHandler,),+ {"server_q": server})+ httpd = ThreadingHTTPServer((host, port), handler)+ httpd.serve_forever()+++def run_server(server: QueueServer, host: str = "127.0.0.1",+ port: int = 8000, use_flask: bool | None = None):+ """Run the HTTP server in the foreground (blocking)."""+ if use_flask is None:+ use_flask = _HAS_FLASK+ if use_flask:+ app = create_app(server)+ app.run(host=host, port=port, threaded=True)+ else:+ run_stdlib(server, host, port)diff --git a/src/edge0/server/chat.py b/src/edge0/server/chat.pynew file mode 100644index 0000000..3babc68--- /dev/null+++ b/src/edge0/server/chat.py@@ -0,0 +1,174 @@+"""edge0 HTTP server: a single-process, one-model OpenAI-style endpoint.++The engine is exclusive to one request at a time (a single MLX process+can stream one generation well; concurrent requests would thrash the+expert cache). Requests queue on a lock: each generation streams+token-by-token over SSE, and the queue drains in FIFO order.+"""++from __future__ import annotations++import json+import threading+import time+from dataclasses import dataclass, field+from typing import Any++from edge0.engine.base import Edge0Engine+++@dataclass+class ChatMessage:+ role: str+ content: str+++@dataclass+class ChatRequest:+ model: str+ messages: list[ChatMessage]+ temperature: float | None = None+ top_p: float | None = None+ top_k: int | None = None+ max_tokens: int | None = None+ seed: int | None = None+ stream: bool = False+ enable_thinking: bool | None = None+ raw: dict[str, Any] = field(default_factory=dict)+++def parse_chat_request(payload: dict) -> ChatRequest:+ msgs = []+ for m in payload.get("messages", []):+ role = str(m.get("role", "user"))+ content = m.get("content", "")+ if isinstance(content, list): # multi-part content: join text parts+ content = "".join(+ p.get("text", "") for p in content if isinstance(p, dict))+ msgs.append(ChatMessage(role=role, content=str(content)))+ return ChatRequest(+ model=str(payload.get("model", "")),+ messages=msgs,+ temperature=payload.get("temperature"),+ top_p=payload.get("top_p"),+ top_k=payload.get("top_k"),+ max_tokens=payload.get("max_tokens"),+ seed=payload.get("seed"),+ stream=bool(payload.get("stream", False)),+ enable_thinking=payload.get("enable_thinking"),+ raw=payload,+ )+++class ChatSession:+ """Tokenize + chat-template + generate for one request."""++ def __init__(self, engine: Edge0Engine, req: ChatRequest):+ self.engine = engine+ self.req = req+ self._tok = engine._tok++ def prompt_ids(self) -> list[int]:+ tok = self._tok+ # ling-mlx-server parity (THINK_FIX_20260907.md): families with a+ # vendored chat template (ling ``engine.encode_chat`` renders the+ # deployment chat_template.jinja with enable_thinking) MUST go+ # through it — the tokenizer's own apply_chat_template renders a+ # DIFFERENT prompt (no think-mode system line, no ``<think>``+ # opener), which is exactly the wrong-template bug.+ encode = getattr(self.engine, "encode_chat", None)+ if callable(encode):+ think = self.req.enable_thinking+ if think is None:+ think = getattr(self.engine, "think", False)+ return encode(+ [m.__dict__ for m in self.req.messages], think=bool(think))+ if hasattr(tok, "apply_chat_template"):+ # Deployment parity with the correct demo (serve.py+ # encode_chat): the qwen35 template's default IS the thinking+ # mode (<think> opener). Passing enable_thinking=False on+ # this template renders an EMPTY <think>\n\n</think> block+ # instead of omitting thinking — a different prompt than the+ # demo, so it must not be passed.+ text = tok.apply_chat_template(+ [m.__dict__ for m in self.req.messages],+ tokenize=False, add_generation_prompt=True)+ else:+ text = self._chat_text()+ ids = tok.encode(text)+ if not ids:+ ids = [tok.bos_token_id or 0]+ return ids++ def _chat_text(self) -> str:+ parts = []+ for m in self.req.messages:+ if m.role == "system":+ parts.append(f"<|im_start|>system\n{m.content}<|im_end|>\n")+ elif m.role == "user":+ parts.append(f"<|im_start|>user\n{m.content}<|im_end|>\n")+ elif m.role == "assistant":+ parts.append(f"<|im_start|>assistant\n{m.content}<|im_end|>\n")+ parts.append("<|im_start|>assistant\n")+ return "".join(parts)++ def gen_config(self):+ from edge0.config import GenerationConfig+ base = getattr(self.engine.cfg, "gen", GenerationConfig())+ kw = {}+ for k in ("temperature", "top_p", "top_k", "max_new_tokens", "seed"):+ v = getattr(self.req, {+ "max_new_tokens": "max_tokens",+ }.get(k, k), None)+ if v is not None:+ kw[k] = v+ return GenerationConfig(**{**base.__dict__, **kw})++ def run(self, on_token=None) -> tuple[list[int], dict]:+ t0 = time.perf_counter()+ # ling-mlx-server parity (THINK_FIX_20260907.md §3.1): clean+ # per-request state. A previous degenerate/truncated turn leaves+ # bad pregate cross-token state and KV behind, which makes every+ # later request collapse from its first token.+ self.engine.reset()+ ids = self.prompt_ids()+ gen = self.gen_config()+ tokens = self.engine.generate(+ ids, gen_config=gen, on_token=on_token)+ usage = {+ "prompt_tokens": len(ids),+ "completion_tokens": len(tokens),+ "total_tokens": len(ids) + len(tokens),+ }+ meta = {"wall_s": round(time.perf_counter() - t0, 3)}+ return tokens, {"usage": usage, **meta}+++class QueueServer:+ """Single-slot serving loop: serialize generations, stream via callback."""++ def __init__(self, engine: Edge0Engine, model_name: str | None = None):+ self.engine = engine+ self.model_name = model_name or engine.name+ self._lock = threading.Lock()++ def health(self) -> dict:+ st = self.engine.stats()+ return {"status": "ok", "model": self.model_name,+ "pos": self.engine.pos}++ def chat(self, req: ChatRequest, on_token=None) -> dict:+ with self._lock:+ sess = ChatSession(self.engine, req)+ return sess.run(on_token=on_token)+++def sse_format(data: dict) -> str:+ return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"+++def decode_tokens(engine: Edge0Engine, tokens: list[int]) -> str:+ tok = engine._tok+ if tok is None:+ return ""+ return tok.decode(tokens)diff --git a/src/edge0/streaming/__init__.py b/src/edge0/streaming/__init__.pynew file mode 100644index 0000000..8e412b0--- /dev/null+++ b/src/edge0/streaming/__init__.py@@ -0,0 +1,17 @@+"""Streaming expert offload: mmap stores, shared LRU caches, staged+decode slots, whole-layer prefill and hot-expert pins."""++from edge0.streaming.cache import PrefetchBuffer, SharedExpertCache+from edge0.streaming.install import install_streaming_experts+from edge0.streaming.layer import StreamingSwitchGLU+from edge0.streaming.mmap import SafetensorsMmap+from edge0.streaming.options import LayerOptions++__all__ = [+ "SafetensorsMmap",+ "SharedExpertCache",+ "PrefetchBuffer",+ "StreamingSwitchGLU",+ "LayerOptions",+ "install_streaming_experts",+]diff --git a/src/edge0/streaming/cache.py b/src/edge0/streaming/cache.pynew file mode 100644index 0000000..d426768--- /dev/null+++ b/src/edge0/streaming/cache.py@@ -0,0 +1,81 @@+"""Expert-bundle caches shared by all streaming MoE layers.++* ``SharedExpertCache`` — global LRU across layers (one shared memory+ budget: a layer that thrashes evicts entries from other layers, which is+ exactly the behavior that keeps the whole-model budget flat).+* ``PrefetchBuffer`` — cap-bounded buffer of eagerly prefetched bundles+ that have NOT yet been promoted into the LRU.+"""++from __future__ import annotations++import threading+from collections import OrderedDict+++class SharedExpertCache:+ """Global LRU across all MoE layers (shared memory budget)."""++ def __init__(self, slots: int):+ self.slots = max(0, slots)+ self._cache = OrderedDict()+ self._lock = threading.Lock()++ def get(self, key):+ with self._lock:+ v = self._cache.get(key)+ if v is not None:+ self._cache.move_to_end(key)+ return v++ def peek(self, key):+ with self._lock:+ return self._cache.get(key)++ def put(self, key, bundle):+ with self._lock:+ self._cache[key] = bundle+ if self.slots > 0 and len(self._cache) > self.slots:+ self._cache.popitem(last=False)++ def pop(self, key):+ with self._lock:+ return self._cache.pop(key, None)++ def size(self):+ with self._lock:+ return len(self._cache)+++class PrefetchBuffer:+ """Cap-bounded staging area for prefetched bundles.++ ``put`` evicts the oldest entry when over capacity (counted as+ ``prefetch_wasted`` by the caller — the build was spent, the bundle was+ never used). ``pop`` removes a bundle when the layer actually consumes+ it (it is then promoted into the shared LRU by the caller).+ """++ def __init__(self, cap: int = 48):+ self.cap = max(0, cap)+ self._buf = OrderedDict()+ self._lock = threading.Lock()++ def put(self, key, bundle):+ with self._lock:+ self._buf[key] = bundle+ evicted = None+ if self.cap > 0 and len(self._buf) > self.cap:+ evicted = self._buf.popitem(last=False)+ return evicted[0] if evicted is not None else None++ def pop(self, key):+ with self._lock:+ return self._buf.pop(key, None)++ def contains(self, key) -> bool:+ with self._lock:+ return key in self._buf++ def set_cap(self, cap: int):+ self.cap = max(0, cap)diff --git a/src/edge0/streaming/install.py b/src/edge0/streaming/install.pynew file mode 100644index 0000000..37b0e7e--- /dev/null+++ b/src/edge0/streaming/install.py@@ -0,0 +1,106 @@+"""Install streaming MoE blocks into a loaded base model.++Generalizes the deployment's ``install_streaming_experts``: instead of+hard-coding a walk of ``model.layers``, the walk is driven by ``MoESpec``+(``block_path`` template + num layers). The returned list of+``StreamingSwitchGLU`` objects is what the engine, prerouter stager, and+prefill hooks drive.+"""++from __future__ import annotations++from edge0.backends import core++from edge0.moe.spec import MoESpec+from edge0.streaming.cache import PrefetchBuffer, SharedExpertCache+from edge0.streaming.layer import StreamingSwitchGLU+from edge0.streaming.mmap import SafetensorsMmap+from edge0.streaming.options import LayerOptions+++def install_streaming_experts(+ model,+ shards: list[SafetensorsMmap],+ spec: MoESpec,+ options: LayerOptions | None = None,+ num_layers: int | None = None,+ shared_cache: SharedExpertCache | None = None,+ prefetch_buffer: PrefetchBuffer | None = None,+) -> list[StreamingSwitchGLU]:+ """Replace every routed-expert block in ``model`` with its streaming+ twin and return the twins in layer order.++ ``num_layers`` defaults to ``spec``-independent discovery: the block+ path template is probed with increasing layer indices until it stops+ resolving. The original blocks are kept on the objects as+ ``_edge0_resident`` (used by the bit-parity tests); nothing else in the+ base model is touched.+ """+ if num_layers is None:+ n = 0+ while True:+ try:+ spec.block_of(model, n)+ except AttributeError:+ break+ n += 1+ if n == 0:+ raise ValueError(+ f"block_path {spec.block_path!r} resolves no layer 0 — "+ "check MoESpec.block_path")+ num_layers = n++ opts = options or LayerOptions()+ cache = shared_cache or SharedExpertCache(opts.cache_slots)+ buf = prefetch_buffer or PrefetchBuffer(opts.prefetch_cap)++ stream_layers: list[StreamingSwitchGLU | None] = []+ for i in range(num_layers):+ block = spec.block_of(model, i)+ if not hasattr(block, "experts") and not hasattr(+ block, "switch_mlp"):+ # Dense (non-routed) layer — nothing to stream.+ stream_layers.append(None)+ continue+ twin = StreamingSwitchGLU(+ shards, i, spec, options=opts,+ shared_cache=cache, prefetch_buffer=buf)+ # keep the resident block reachable (bit-parity tests, debugging)+ block._edge0_resident = block+ # swap the twin into the block, following the family convention:+ # qwen-style blocks call ``switch_mlp(x, inds)``; ling-style blocks+ # hold ``experts = SwitchGLU(...)`` and call ``experts(x, idx)``.+ # The original submodule is stashed for parity tests.+ if hasattr(block, "switch_mlp"):+ block._edge0_resident_switch = block.switch_mlp+ block.switch_mlp = twin+ else:+ block._edge0_resident_switch = block.experts+ block.experts = twin+ if opts.top_k is not None:+ res = getattr(block, "_edge0_resident", block)+ if hasattr(res, "top_k"):+ res.top_k = opts.top_k+ stream_layers.append(twin)+ return stream_layers+++def attach_streaming(block, twin) -> None:+ """Point a MoE block's ``__call__`` at its streaming twin.++ For vendored models whose layer code calls ``self.mlp(...)`` directly,+ the installer swaps the block object itself (``setattr`` on the layer);+ for models whose blocks carry a ``__call__`` the engine may patch it+ class-level. This helper exists so both paths share one spelling.+ """+ twin._edge0_resident = getattr(block, "_edge0_resident", block)+ block._edge0_stream = twin+++def route_indices(mx_array) -> core.array:+ """Sanity guard: indices must be a GPU mx array when they leave the+ block (the whole staged design assumes no per-layer host syncs)."""+ assert isinstance(mx_array, core.array), (+ "router indices must stay on GPU (core.array), got "+ f"{type(mx_array).__name__}")+ return mx_arraydiff --git a/src/edge0/streaming/layer.py b/src/edge0/streaming/layer.pynew file mode 100644index 0000000..9ac9199--- /dev/null+++ b/src/edge0/streaming/layer.py@@ -0,0 +1,1395 @@+"""StreamingSwitchGLU: the SSD-offloaded MoE expert block.++Faithful port of the deployment's ``streaming_experts_qwen35.py``+(itself a port of the ling ``stream_experts.py`` design), generalized so+ONE class serves every supported model:++* expert weight layout (separate vs fused gate+up projections) and+ safetensors key prefix come from ``MoESpec``;+* every behavior knob (staged decode, hot pins, prefetch, compile,+ incremental stack, whole-layer prefill) comes from ``LayerOptions``+ instead of env vars.++Public surface consumed by the engine / prerouter / prefill hooks:++ load_full_layer / clear_full_layer / load_hot_layer /+ materialize_hot / dematerialize_hot / clear_hot_layer+ stage_experts / wait_staged / swap_staged / sync_actuals /+ stage_from_prefill / stage_from_history+ prefetch / prefetch_all / warm_pages+ stats / reset++The ``__call__(x, indices)`` routing:++1. whole-layer prefill path — full stacked weights resident+ (``_full_weights``), sorted gather, bit-identical to the resident+ math; also records usage + last-token top-k for staging.+2. hot-stack prefill path — resident top-N stack+ (``_hot_weights``) with exact per-expert scatter-add for misses.+3. staged decode path — routing through fixed staged slots with+ a single GPU-side expert->slot table (``core.take``), no per-layer+ host syncs; missing experts map to an all-zero overflow row+ (production prerouter consumers have zero misses by construction:+ the staged set IS the prerouter's prediction).+4. exact decode path — per-expert bundles (pinned / shared LRU+ / prefetch buffer / pool builds), sorted gather for large batches.+"""++from __future__ import annotations++import threading+import time+from concurrent.futures import ThreadPoolExecutor+from functools import partial++from edge0.backends import core, quant+from edge0.backends import nn+import numpy as np++from mlx_lm.models.switch_layers import _gather_sort, _scatter_unsort++from edge0.moe.spec import MoESpec+from edge0.streaming.cache import PrefetchBuffer, SharedExpertCache+from edge0.streaming.mmap import SafetensorsMmap, u32_view+from edge0.streaming.options import LayerOptions+++def _swiglu(up, gate):+ return nn.silu(gate) * up+++class StreamingSwitchGLU:+ """Streaming version of a routed-expert GLU block.++ Only the routed experts are streamed. Shared experts, routers, and+ attention stay resident in the base model.+ """++ def __init__(+ self,+ shards: list[SafetensorsMmap],+ layer_idx: int,+ spec: MoESpec,+ options: LayerOptions | None = None,+ shared_cache: SharedExpertCache | None = None,+ prefetch_buffer: PrefetchBuffer | None = None,+ ):+ self.layer_idx = layer_idx+ self.spec = spec+ self.num_experts = spec.num_experts+ self.group_size = spec.quant.group_size+ self.bits = spec.quant.bits+ self.mode = spec.quant.mode+ self._fuse_gu = spec.fuse_gu+ self._bundle_projs = spec.bundle_projs++ o = options or LayerOptions()+ self._staged_mode = o.staged+ self._staged_replace = o.staged_replace+ self._staged_n = max(1, o.staged_n)+ # Router top-k that triggers the staged path (single-token decode);+ # set by the engine to the routed top_k (usually the staged_n).+ self._staged_trigger = o.staged_trigger+ self._staged_sync = o.staged_sync+ self.hot_per_layer = o.hot_per_layer+ self.hot_update_interval = o.hot_update_interval+ self.hot_decay = o.hot_decay+ self._pin_bonus = o.pin_bonus+ self._use_compile = o.use_compile++ self._pool = ThreadPoolExecutor(max_workers=o.load_threads)+ self._prefetch_pool = ThreadPoolExecutor(+ max_workers=max(1, o.prefetch_threads))++ self._shards = shards+ self._shard_idx = {+ name: i+ for i, s in enumerate(shards)+ for name in s.entries+ }+ self._prefix = spec.key_template.format(layer=layer_idx)+ self._shape = {}+ for proj in ("gate_proj", "up_proj", "down_proj"):+ for part in ("weight", "scales", "biases"):+ name = f"{self._prefix}.{proj}.{part}"+ self._shape[(proj, part)] = self._shards[+ self._shard_idx[name]].entries[name]["shape"]+ # Bundle-format shapes: what _build produces / caches store / math+ # consumes. Fused gate_up = gate rows on top of up rows along axis 1.+ if self._fuse_gu:+ self._bundle_shape = {}+ for part in ("weight", "scales", "biases"):+ gs = self._shape[("gate_proj", part)]+ self._bundle_shape[("gate_up_proj", part)] = (+ gs[0], 2 * gs[1]) + tuple(gs[2:])+ self._bundle_shape[("down_proj", part)] = self._shape[+ ("down_proj", part)]+ else:+ self._bundle_shape = self._shape++ self._pinned = {}+ self._hot_counts = {}+ self._calls = 0+ self._full_weights: dict | None = None+ self._hot_weights: dict | None = None+ self._hot_backing: dict | None = None+ self._hot_key: tuple | None = None+ self._hot_set: set | None = None+ self.predicted_experts: set[int] = set()+ self._lock = threading.Lock()+ self.last_used: list[int] = []+ self._stats = {"loads": 0, "hits": 0, "load_wall": 0.0}+ self.shared_cache = shared_cache or SharedExpertCache(o.cache_slots)+ self._prefetch_buf = prefetch_buffer or PrefetchBuffer(o.prefetch_cap)+ self._inflight: dict[int, object] = {}+ self._stats.update({+ "prefetch_submitted": 0,+ "prefetch_hits": 0,+ "prefetch_wasted": 0,+ "prefetch_wait": 0.0,+ })++ # ---- MoE math (core.compile variants) ------------------------------++ def _make_moe_math():+ gs, bt, md = self.group_size, self.bits, self.mode++ @core.compile+ def fn(x, w_u, s_u, b_u, w_g, s_g, b_g, w_d, s_d, b_d, local):+ xe = core.expand_dims(x, (-2, -3))+ x_up = quant.gather_qmm(+ xe, w_u, s_u, b_u, rhs_indices=local,+ transpose=True, group_size=gs, bits=bt, mode=md,+ sorted_indices=False)+ x_gate = quant.gather_qmm(+ xe, w_g, s_g, b_g, rhs_indices=local,+ transpose=True, group_size=gs, bits=bt, mode=md,+ sorted_indices=False)+ z = quant.gather_qmm(+ _swiglu(x_up, x_gate), w_d, s_d, b_d,+ rhs_indices=local, transpose=True, group_size=gs,+ bits=bt, mode=md, sorted_indices=False)+ return z.squeeze(-2)+ return fn++ def _make_moe_math_sorted():+ gs, bt, md = self.group_size, self.bits, self.mode++ @core.compile+ def fn(x, w_u, s_u, b_u, w_g, s_g, b_g, w_d, s_d, b_d,+ local, inv_order, out_shape):+ x_up = quant.gather_qmm(+ x, w_u, s_u, b_u, rhs_indices=local,+ transpose=True, group_size=gs, bits=bt, mode=md,+ sorted_indices=True)+ x_gate = quant.gather_qmm(+ x, w_g, s_g, b_g, rhs_indices=local,+ transpose=True, group_size=gs, bits=bt, mode=md,+ sorted_indices=True)+ z = quant.gather_qmm(+ _swiglu(x_up, x_gate), w_d, s_d, b_d,+ rhs_indices=local, transpose=True, group_size=gs,+ bits=bt, mode=md, sorted_indices=True)+ return _scatter_unsort(z, inv_order, out_shape).squeeze(-2)+ return fn++ def _make_moe_math_fused():+ gs, bt, md = self.group_size, self.bits, self.mode++ @core.compile+ def fn(x, w_gu, s_gu, b_gu, w_d, s_d, b_d, local):+ xe = core.expand_dims(x, (-2, -3))+ x_gu = quant.gather_qmm(+ xe, w_gu, s_gu, b_gu, rhs_indices=local,+ transpose=True, group_size=gs, bits=bt, mode=md,+ sorted_indices=False)+ x_gate, x_up = core.split(x_gu, 2, axis=-1)+ z = quant.gather_qmm(+ _swiglu(x_up, x_gate), w_d, s_d, b_d,+ rhs_indices=local, transpose=True, group_size=gs,+ bits=bt, mode=md, sorted_indices=False)+ return z.squeeze(-2)+ return fn++ def _make_moe_math_sorted_fused():+ gs, bt, md = self.group_size, self.bits, self.mode++ @core.compile+ def fn(x, w_gu, s_gu, b_gu, w_d, s_d, b_d,+ local, inv_order, out_shape):+ x_gu = quant.gather_qmm(+ x, w_gu, s_gu, b_gu, rhs_indices=local,+ transpose=True, group_size=gs, bits=bt, mode=md,+ sorted_indices=True)+ x_gate, x_up = core.split(x_gu, 2, axis=-1)+ z = quant.gather_qmm(+ _swiglu(x_up, x_gate), w_d, s_d, b_d,+ rhs_indices=local, transpose=True, group_size=gs,+ bits=bt, mode=md, sorted_indices=True)+ return _scatter_unsort(z, inv_order, out_shape).squeeze(-2)+ return fn++ self._moe_math = _make_moe_math()+ self._moe_math_sorted = _make_moe_math_sorted()+ if self._fuse_gu:+ # Fused gate+up: replace the math fns with the fused variants+ # (same pattern as the original streaming_experts_qwen35.py) —+ # every call site passes its wargs through unchanged.+ self._moe_math = _make_moe_math_fused()+ self._moe_math_sorted = _make_moe_math_sorted_fused()++ # ---- staged decode ------------------------------------------------+ # Port of ling stream_experts.py R1 fixed-slot staging. The router's+ # indices stay on GPU: a CPU-built [num_experts] expert->slot table+ # maps them into the staged tensor via core.take, so a whole decode+ # step builds/evaluates with a single sync (the final logits eval)+ # instead of 40 per-layer tolist barriers. Missing experts map to+ # the overflow zero slot (contribution dropped) — with a prerouter+ # the staged set IS the routing, so this never fires in production.+ # Double-buffered staged slots: `_staged_state` is the ACTIVE set+ # read by the current decode step's __call__ (never disturbed+ # mid-step); `_staged_state_next` is the fill target written by+ # stage_experts, swapped in at the step boundary by swap_staged().+ self._staged_state: tuple | None = None+ self._staged_state_next: tuple | None = None+ self._staging_lock = threading.Lock()+ self._staging_future = None+ self._staged_pending: list[int] | None = None+ self._pending_indices: core.array | None = None+ self._staged_at_use: set[int] | None = None+ self._last_prefill_topk: list[int] | None = None+ # Small cache of already-materialized (GPU) expert->slot tables.+ self._slot_table_cache: dict[tuple, core.array] = {}+ # Cache of the assembled (slot_of, stacked-wargs) pair per staged+ # expert set, so decode steps whose staged set repeats don't rebuild+ # the 9 core.stack nodes + slot table on every layer every step.+ if not o.asm_cache:+ class _NoAsmCache(dict):+ def __setitem__(self, k, v): # never store+ pass+ self._staged_asm_cache = _NoAsmCache()+ else:+ self._staged_asm_cache = {}+ # Persistent all-zero overflow slot rows (slot index = _staged_n),+ # so missing experts map to a real zero row.+ self._zero_slot = {}+ for proj in self._bundle_projs:+ for part in ("weight", "scales", "biases"):+ shape = self._bundle_shape[(proj, part)][1:]+ dtype = core.uint32 if part == "weight" else core.bfloat16+ self._zero_slot[(proj, part)] = core.zeros(shape, dtype)+ # ---- incremental stack --------------------------------------------+ # Sticky-slot persistent staged tensors: the [n+1, ...] stacked+ # weights are allocated ONCE per layer and only the slots whose+ # expert changed are rewritten in place each step, replacing the+ # 9 core.stack graph nodes per layer per step. Safe only when the+ # previous forward's eval has finished (staged_sync, the+ # production default) — the async double-buffered fill would race+ # the writes.+ self._incr_mode = (+ o.incr_stack and self._staged_mode and self._staged_sync+ and not self._staged_replace)+ # Write-back LRU (dedup staged set from the shared LRU): while+ # staged, an expert's bundle is held only by the incremental stack+ # machinery; it enters the LRU only when evicted from the stack.+ self._incr_writeback = (+ self._incr_mode and o.incr_writeback)+ self._incr_tensors = None # {(proj,part): core.array [n+1, ...]}+ self._incr_slot_table = None # core.array [num_experts] int32+ self._incr_occupancy = None # list[slot] -> expert id | None+ self._incr_bundles = {} # expert -> bundle (current set)+ self._stats.update({+ "staged_used": 0,+ "staged_dropped": 0,+ "staged_fallback": 0,+ "stage_wall": 0.0,+ "stage_wait_ms": 0.0,+ "stage_delta_hits": 0,+ "stage_cache_hits": 0,+ "stage_builds": 0,+ "incr_writes": 0,+ })++ # ---- loading ---------------------------------------------------------++ def _shard_for(self, name: str) -> SafetensorsMmap:+ return self._shards[self._shard_idx[name]]++ def _build(self, expert: int):+ """Load one expert's 9 tensors (gate/up/down x weight/scales/biases).++ Hot-stack fast path: if the expert is a member of the resident+ hot stack, slice its rows from the stack instead of reading the+ shard — zero IO, zero page-cache pressure."""+ if self._hot_backing is not None and expert in self._hot_set:+ idx = self._hot_key.index(expert)+ shape = self._bundle_shape+ b = {}+ for (proj, part), buf in self._hot_backing.items():+ sh = shape[(proj, part)]+ per = buf.size // len(self._hot_key)+ sl = buf[idx * per:(idx + 1) * per]+ if part == "weight":+ b[(proj, part)] = core.array(+ sl.view("<u4").reshape(sh[1:]))+ else:+ b[(proj, part)] = core.array(sl.view("<u2")).view(+ core.bfloat16).reshape(sh[1:])+ return b+ per_w = {}++ def _read_rows(proj, part):+ name = f"{self._prefix}.{proj}.{part}"+ raw = self._shard_for(name).raw(name)+ shape = self._shape[(proj, part)]+ per = raw.size // self.num_experts+ return raw[expert * per:(expert + 1) * per]++ def _to_mx(sl, key):+ part = key[1]+ shape = self._bundle_shape[key]+ if part == "weight":+ return core.array(u32_view(sl, shape[1:]))+ return core.array(sl.view("<u2")).view(+ core.bfloat16).reshape(shape[1:])++ if self._fuse_gu:+ # gate rows on top of up rows (matches split(x_gu, 2) order);+ # numpy-level concat of the raw byte slices before the single+ # core.array per part (pool thread, off the critical path).+ for part in ("weight", "scales", "biases"):+ sl = np.concatenate([+ _read_rows("gate_proj", part),+ _read_rows("up_proj", part)])+ per_w[("gate_up_proj", part)] = _to_mx(+ sl, ("gate_up_proj", part))+ for part in ("weight", "scales", "biases"):+ per_w[("down_proj", part)] = _to_mx(+ _read_rows("down_proj", part), ("down_proj", part))+ return per_w+ for proj in ("gate_proj", "up_proj", "down_proj"):+ for part in ("weight", "scales", "biases"):+ per_w[(proj, part)] = _to_mx(+ _read_rows(proj, part), (proj, part))+ return per_w++ def warm_pages(self, experts):+ """Fault expert mmap pages into the OS page cache without retaining+ mx arrays (MLX active memory stays flat)."""+ for e in experts:+ for proj in ("gate_proj", "up_proj", "down_proj"):+ for part in ("weight", "scales", "biases"):+ name = f"{self._prefix}.{proj}.{part}"+ raw = self._shard_for(name).raw(name)+ per = raw.size // self.num_experts+ sl = raw[e * per:(e + 1) * per]+ _ = sl.sum()++ def _get_bundles(self, experts):+ """Resolve bundles: pinned/LRU hits synchronously, misses via pool."""+ bundles = {}+ missing = []+ with self._lock:+ for e in experts:+ p = self._pinned.get(e)+ if p is not None:+ bundles[e] = p+ continue+ bundles[e] = None+ missing.append(e)+ still_missing = []+ for e in missing:+ cached = self.shared_cache.get((self.layer_idx, e))+ if cached is not None:+ self._stats["hits"] += 1+ bundles[e] = cached+ else:+ still_missing.append(e)+ missing = still_missing+ resolved = set()+ for e in missing:+ with self._lock:+ bundle = self._prefetch_buf.pop((self.layer_idx, e))+ if bundle is not None:+ self._stats["prefetch_hits"] += 1+ self._stats["hits"] += 1+ with self._lock:+ self._hot_counts[e] = (+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)+ self.shared_cache.put((self.layer_idx, e), bundle)+ bundles[e] = bundle+ resolved.add(e)+ missing = [e for e in missing if e not in resolved]+ resolved = set()+ for e in missing:+ with self._lock:+ fut = self._inflight.get(e)+ if fut is not None:+ t0 = time.perf_counter()+ tensors = fut.result()+ with self._lock:+ self._inflight.pop(e, None)+ self._stats["prefetch_wait"] += time.perf_counter() - t0+ self._hot_counts[e] = (+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)+ self._stats["prefetch_hits"] += 1+ self._stats["hits"] += 1+ self.shared_cache.put((self.layer_idx, e), tensors)+ bundles[e] = tensors+ resolved.add(e)+ missing = [e for e in missing if e not in resolved]+ if missing:+ t0 = time.perf_counter()+ built = list(self._pool.map(self._build, missing))+ wall = time.perf_counter() - t0+ with self._lock:+ for e, tensors in zip(missing, built):+ self._stats["loads"] += 1+ self._hot_counts[e] = (+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)+ bundles[e] = tensors+ self._stats["load_wall"] += wall+ for e, tensors in zip(missing, built):+ self.shared_cache.put((self.layer_idx, e), tensors)+ return bundles++ # ---- prefetch --------------------------------------------------------++ def prefetch(self, experts):+ unique = sorted({int(e) for e in experts})+ if not unique:+ return+ submitted = []+ with self._lock:+ missing = []+ for e in unique:+ if (+ e in self._pinned+ or e in self._inflight+ or self.shared_cache.peek((self.layer_idx, e)) is not None+ or self._prefetch_buf.contains((self.layer_idx, e))+ ):+ continue+ missing.append(e)+ for e in missing:+ fut = self._prefetch_pool.submit(self._build, e)+ self._inflight[e] = fut+ submitted.append((e, fut))+ self._stats["prefetch_submitted"] += len(missing)+ # Register the done-callback OUTSIDE the lock: if the build finished+ # before submit() returned, add_done_callback fires synchronously on+ # this thread, and _on_prefetch_done re-acquires the same non-reentrant+ # lock — deadlock. Moving the registration out fixes it.+ for e, fut in submitted:+ fut.add_done_callback(partial(self._on_prefetch_done, e))++ def prefetch_all(self):+ n = self.num_experts+ if self._prefetch_buf.cap < n + 32:+ self._prefetch_buf.set_cap(n + 32)+ self.prefetch(range(n))++ def _on_prefetch_done(self, expert, fut):+ try:+ bundle = fut.result()+ except Exception:+ with self._lock:+ self._inflight.pop(expert, None)+ return+ with self._lock:+ self._inflight.pop(expert, None)+ if self.shared_cache.peek((self.layer_idx, expert)) is not None:+ return+ evicted = self._prefetch_buf.put((self.layer_idx, expert), bundle)+ if evicted is not None:+ self._stats["prefetch_wasted"] += 1++ # ---- fixed-slot staging (staged decode) ------------------------------++ def _build_staged_bundles(self, experts: list[int]):+ """Resolve per-expert bundles for the next staged slots.++ Runs on the thread pool. Reuses the previous staged state by+ reference (delta reuse), then pinned / shared-LRU mx arrays; only+ genuinely cold experts go through the raw mmap build. The core.stack+ into fixed [n+1, ...] tensors is done on the main thread in __call__+ because MLX lazy-op construction is not safe concurrently with eval.+ """+ _t0 = time.perf_counter()+ n = self._staged_n+ exp = sorted({int(e) for e in experts})[:n]+ prev = {}+ st = self._staged_state+ if st is not None:+ pbundles, _, pexp = st+ for i, e in enumerate(pexp):+ prev[e] = pbundles[i]+ slot_of = [n] * self.num_experts+ for i, e in enumerate(exp):+ slot_of[e] = i+ bundles = []+ for e in exp:+ b = prev.get(e)+ if b is not None:+ with self._lock:+ self._stats["stage_delta_hits"] += 1+ if b is None:+ with self._lock:+ b = self._pinned.get(e)+ if b is None:+ b = self.shared_cache.get((self.layer_idx, e))+ if b is not None:+ with self._lock:+ self._stats["stage_cache_hits"] += 1+ if b is None:+ b = self._build(e)+ # put the fresh build into the shared LRU so later steps+ # hit the cache instead of rebuilding: the staged resolver+ # above checks shared_cache BEFORE _build, so this closes+ # the loop (delta miss -> LRU hit, no rebuild).+ self.shared_cache.put((self.layer_idx, e), b)+ with self._lock:+ self._stats["stage_builds"] += 1+ bundles.append(b)+ wall = time.perf_counter() - _t0+ return tuple(bundles), slot_of, exp, wall++ def _stage_incr(self, experts: list[int]):+ """Incremental-stack resolver: sticky slot assignment + in-place+ delta writes into the persistent stacked tensors.++ Keeps every surviving expert in its current slot (zero work), frees+ the slots of evicted experts, and assigns newly arriving experts to+ the freed slots — writing ONLY those slots' rows in place (one+ __setitem__ per (proj, part) per changed slot) plus slot-table+ updates. Bundle resolution cascade is identical to+ _build_staged_bundles (pinned -> shared LRU -> raw build).+ Runs on the main thread (sync staged mode only)."""+ _t0 = time.perf_counter()+ n = self._staged_n+ exp = sorted({int(e) for e in experts})[:n]+ exp_set = set(exp)++ # Lazy one-time allocation of the persistent tensors. Slot n is the+ # all-zero overflow row and is NEVER written after this init.+ if self._incr_tensors is None:+ self._incr_tensors = {}+ for proj in self._bundle_projs:+ for part in ("weight", "scales", "biases"):+ shape = self._bundle_shape[(proj, part)][1:]+ dtype = core.uint32 if part == "weight" else core.bfloat16+ t = core.zeros((n + 1, *shape), dtype)+ core.eval(t)+ self._incr_tensors[(proj, part)] = t+ self._incr_slot_table = core.full(+ (self.num_experts,), n, dtype=core.int32)+ core.eval(self._incr_slot_table)+ self._incr_occupancy = [None] * n++ occ = self._incr_occupancy+ # Sticky assignment: survivors keep slots; evicted slots free up.+ free = [i for i in range(n) if occ[i] is not None+ and occ[i] not in exp_set]+ for i in free:+ occ[i] = None+ free = [i for i in range(n) if occ[i] is None]+ # slot-of list for compatibility consumers (_staged_state / stats).+ slot_of_list = [n] * self.num_experts+ for i in range(n):+ e = occ[i]+ if e is not None:+ slot_of_list[e] = i++ changed = [e for e in exp if e not in self._incr_bundles]+ new_slots = []+ for j, e in enumerate(changed):+ slot = free[j] if j < len(free) else None+ new_slots.append((e, slot))++ writes = []+ # Mirror the occupancy exactly: only the CURRENT staged set keeps+ # bundles. A stale entry would hide a returning expert from+ # `changed` (no slot assigned, no row write -> its contribution+ # silently maps to the zero row).+ bundles = {e: self._incr_bundles[e] for e in exp+ if e in self._incr_bundles}+ for e, slot in new_slots:+ if slot is None:+ # |exp| <= n guarantees a free slot for every changed+ # expert (survivors <= |exp|), so this is unreachable;+ # guard anyway: drop the expert rather than corrupt a slot.+ continue+ b = self._pinned.get(e)+ if b is None:+ b = self.shared_cache.get((self.layer_idx, e))+ if b is not None:+ with self._lock:+ self._stats["stage_cache_hits"] += 1+ if b is None:+ b = self._build(e)+ if not self._incr_writeback:+ # eager mode: share the fresh build via the LRU+ self.shared_cache.put((self.layer_idx, e), b)+ with self._lock:+ self._stats["stage_builds"] += 1+ bundles[e] = b+ occ[slot] = e+ slot_of_list[e] = slot+ writes.append((slot, b))+ # Evicted experts: table back to the zero overflow slot.+ evicted = [e for e in self._incr_bundles if e not in exp_set]+ if self._incr_writeback:+ # Write-back LRU: while an expert is staged its canonical bundle+ # lives ONLY in _incr_bundles (the persistent stack rows are the+ # math copy) — no LRU duplication of the staged set. On eviction+ # from the stack the bundle is written back so returning experts+ # still hit the LRU.+ for e in evicted:+ self.shared_cache.put((self.layer_idx, e),+ self._incr_bundles[e])+ self._incr_bundles = bundles++ # ---- in-place delta writes (main thread, previous graph already+ # evaluated: stage runs at the step boundary in sync mode) ----+ for slot, b in writes:+ for proj in self._bundle_projs:+ for part in ("weight", "scales", "biases"):+ self._incr_tensors[(proj, part)][slot] = b[(proj, part)]+ tbl = self._incr_slot_table+ for e in evicted:+ tbl[e] = n+ for e, slot in new_slots:+ if slot is not None:+ tbl[e] = slot+ # FIX(2026-09-02): force-evaluate the slot table after in-place+ # mutations so the consuming forward's lazy graph (core.take /+ # _moe_math) reads the UPDATED mapping — without this, near-100%+ # slot churn reads stale slot tables and the output collapses.+ if writes or evicted or new_slots:+ core.eval(self._incr_slot_table)+ with self._lock:+ self._stats["incr_writes"] += len(writes)+ self._stats["stage_delta_hits"] += len(exp) - len(writes)+ self._stats["stage_wall"] += time.perf_counter() - _t0++ # Compatibility state: bundles tuple ordered like the old resolver+ # (sync_actuals promotion + fallback paths read it).+ return (tuple(bundles[e] for e in exp), slot_of_list, exp)++ def stage_experts(self, experts):+ """Async-fill the NEXT slot set for the following decode step.++ Writes to `_staged_state_next` (never the active `_staged_state`), so+ the fill for token t+1 can run during token t without disturbing the+ set token t is reading. swap_staged() promotes it at the step+ boundary. If the fill cannot land in time, the consuming __call__+ falls back to the exact per-layer path (staged_fallback).+ """+ if not self._staged_mode:+ return+ unique = sorted({int(e) for e in experts})+ if not unique:+ return+ self._drain_rearm()+ with self._staging_lock:+ if self._staging_future is not None:+ self._staged_pending = unique+ return+ if self._staged_sync:+ if self._incr_mode:+ self._staged_state_next = self._stage_incr(unique)+ else:+ bundles, slot_of, exp, wall = self._build_staged_bundles(+ unique)+ self._staged_state_next = (tuple(bundles), slot_of, exp)+ with self._lock:+ self._stats["stage_wall"] += wall+ return+ self._submit_stage_locked(unique)++ def wait_staged(self):+ """Block until the in-flight staged fill lands, then promote it to+ the active state (ling stream_experts.py:441-455 semantics: the+ fill's completion IS the promotion — no separate blocking+ step-boundary swap; each layer waits lazily at consume time so the+ pool's SSD builds overlap the forward's graph building)."""+ self._drain_rearm()+ with self._staging_lock:+ fut = self._staging_future+ if fut is not None:+ t0 = time.perf_counter()+ try:+ fut.result()+ except Exception:+ pass+ with self._lock:+ self._stats["stage_wait_ms"] += (+ time.perf_counter() - t0) * 1000+ with self._staging_lock:+ if self._staged_state_next is not None:+ self._staged_state = self._staged_state_next+ self._staged_state_next = None++ def swap_staged(self):+ """Promote the filled next set to active at the step boundary.++ Blocks until the in-flight fill lands (the exposed remainder of the+ hidden fill), then promotes. Layers with no pending fill keep their+ current active set (or None on the very first step)."""+ self.wait_staged()+ with self._staging_lock:+ if self._staged_state_next is not None:+ self._staged_state = self._staged_state_next+ self._staged_state_next = None+ self._staged_pending = None++ def _submit_stage_locked(self, unique):+ try:+ fut = self._pool.submit(self._build_staged_bundles, unique)+ except RuntimeError:+ self._staging_future = None+ return+ self._staging_future = fut+ # If the future is ALREADY done (idle pool + tiny build), the done+ # callback fires synchronously INSIDE this call — but we hold+ # _staging_lock here and _on_stage_done needs it => deadlock.+ # Fix: only attach the callback while still holding the lock if the+ # future is pending; for an already-done one, detach-then-handle+ # after release via _staged_rearm (picked up at the next entry).+ if not fut.done():+ fut.add_done_callback(self._on_stage_done)+ return+ self._staged_rearm = fut++ def _drain_rearm(self):+ """Handle a synchronously-completed fill outside _staging_lock."""+ fut = getattr(self, "_staged_rearm", None)+ if fut is None:+ return+ self._staged_rearm = None+ self._on_stage_done(fut)++ def _on_stage_done(self, fut):+ try:+ arrays, slot_of, exp, wall = fut.result()+ except Exception:+ with self._staging_lock:+ self._staging_future = None+ return+ with self._staging_lock:+ self._staged_state_next = (arrays, slot_of, exp)+ self._staging_future = None+ with self._lock:+ self._stats["stage_wall"] += wall+ if self._staged_pending:+ p = self._staged_pending+ self._staged_pending = None+ self._submit_stage_locked(p)++ def sync_actuals(self):+ """After a decode step: record this step's actual router indices so+ they can be staged for the next step (the "current router as the+ prerouter" substitution). No correctness handling: missing experts are+ simply dropped by the staged path."""+ idx = self._pending_indices+ if idx is None:+ return+ try:+ flat = idx.reshape(-1).tolist()+ except Exception:+ return+ self._pending_indices = None+ unique = sorted(set(int(v) for v in flat))+ if not unique:+ return+ with self._lock:+ self.last_used = unique+ for e in unique:+ self._hot_counts[e] = (+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)+ used_exp = self._staged_at_use+ if used_exp is not None:+ dropped = [e for e in unique if e not in used_exp]+ self._stats["staged_dropped"] += len(dropped)+ # Promote bundles the router actually used into the shared LRU+ # so they survive beyond the staging state. Skipped in+ # write-back mode: staged bundles enter the LRU only on stack+ # eviction (avoiding staged-set duplication).+ if self._incr_writeback:+ return+ st = self._staged_state+ if st is not None:+ sbundles, _, sexp = st+ smap = {e: sbundles[i] for i, e in enumerate(sexp)}+ for e in unique:+ if e in self._pinned:+ continue+ b = smap.get(e)+ if b is not None:+ self.shared_cache.put((self.layer_idx, e), b)++ def stage_from_prefill(self):+ """After prefill: stage the last prefill token's actual top-k so the+ first decode step is already staged (router used as the predictor)."""+ if not self._staged_mode:+ return+ topk = getattr(self, "_last_prefill_topk", None)+ if not topk:+ return+ unique = sorted(set(int(v) for v in topk))+ if unique:+ self.last_used = unique+ self.stage_experts(unique)++ def _refresh_hot_pins(self):+ if self.hot_per_layer <= 0:+ return+ cands = set(self._hot_counts) | set(self.predicted_experts)+ scores = {e: self._hot_counts.get(e, 0.0) for e in cands}+ for e in self.predicted_experts:+ scores[e] = scores.get(e, 0.0) + self._pin_bonus+ top = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)+ top = top[:self.hot_per_layer]+ target = {e for e, _ in top}+ with self._lock:+ self._pinned = {e: t for e, t in self._pinned.items()+ if e in target}+ for e in target:+ cached = self.shared_cache.pop((self.layer_idx, e))+ if cached is not None:+ self._pinned[e] = cached+ elif e not in self._pinned:+ self._pinned[e] = self._build(e)+ self._hot_target = target++ def refresh_hot_pins(self):+ if self.hot_per_layer <= 0:+ return+ try:+ self._refresh_hot_pins()+ except Exception: # best-effort+ pass++ def stats(self):+ with self._lock:+ s = dict(self._stats)+ s["pinned"] = len(self._pinned)+ s["shared_size"] = self.shared_cache.size()+ s["calls"] = self._calls+ return s++ # ---- whole-layer prefill (load full layer once, then drop) ------------++ def load_full_layer(self) -> None:+ if self._full_weights is not None:+ return+ # The checkpoint stores experts ALREADY STACKED per layer+ # (switch_mlp.gate_proj.weight [256,512,256], one tensor) — so+ # this is 9 direct whole-tensor loads, NOT 256x9 per-expert+ # builds. The core.array.view bitcast makes it ~9ms/layer; with+ # before_layer_cb + async_eval_per_layer, the CPU load hides+ # under the previous layer's GPU execution.+ w = {}++ def _read_full(proj, part):+ name = f"{self._prefix}.{proj}.{part}"+ raw = self._shard_for(name).raw(name)+ return raw++ for proj in self._bundle_projs:+ for part in ("weight", "scales", "biases"):+ if self._fuse_gu and proj == "gate_up_proj":+ # Fused rows = gate rows on top of up rows PER EXPERT+ # (the [E, 2*INTER, in] layout _build produces): the+ # shard stores gate/up as two separate [E, ...]+ # tensors, so interleave per-expert slices — a flat+ # concatenate of both tensors would misalign rows.+ raw_g = _read_full("gate_proj", part)+ raw_u = _read_full("up_proj", part)+ per = raw_g.size // self.num_experts+ shape = self._bundle_shape[(proj, part)]+ rows = np.concatenate([+ np.concatenate([+ raw_g[e * per:(e + 1) * per],+ raw_u[e * per:(e + 1) * per]])+ for e in range(self.num_experts)])+ if part == "weight":+ arr = core.array(u32_view(rows, shape))+ else:+ arr = core.array(rows.view("<u2")).view(+ core.bfloat16).reshape(shape)+ else:+ raw = _read_full(proj, part)+ shape = self._bundle_shape[(proj, part)]+ if part == "weight":+ arr = core.array(u32_view(raw, shape))+ else:+ arr = core.array(raw.view("<u2")).view(+ core.bfloat16).reshape(shape)+ w[(proj, part)] = arr+ self._full_weights = w++ def clear_full_layer(self) -> None:+ self._full_weights = None++ # ---- hot-layer prefill (LRU-resident partial stack) -------------------+ # Middle state between on-demand (re-reads the union every request)+ # and whole-layer E3b (thrashes the page cache): keep the layer's+ # top-N most frequent experts as a resident stacked tensor; misses go+ # through the exact per-expert path and are scatter-added.++ def load_hot_layer(self, n_hot: int = 128) -> None:+ """Load/refresh the layer's hot-expert weights.++ Backing store = one NUMPY array per (proj, part) holding the+ stacked rows of the top-n_hot experts (page cache, survives across+ requests, counts ZERO toward MLX active). The mx window+ (materialize_hot) exposes a few layers at a time so peak MLX+ memory stays at window_size * n_hot * ~1.2MB instead of+ 40 * n_hot * 1.2MB.++ Selection = top-n_hot by _hot_counts (decayed frequency). The+ backing is rebuilt only when membership changes by more than+ 12.5%; small churn is absorbed by the exact miss path."""+ counts = self._hot_counts+ if counts:+ hot = sorted(counts, key=lambda e: counts[e], reverse=True)+ hot = hot[:n_hot]+ else:+ hot = list(range(n_hot))+ key = tuple(hot)+ if self._hot_backing is not None:+ if self._hot_key == key:+ return+ old_set = set(self._hot_key)+ changed = len([e for e in key if e not in old_set])+ if changed <= max(1, n_hot // 8):+ return+ backing = {}+ for proj in self._bundle_projs:+ for part in ("weight", "scales", "biases"):+ if self._fuse_gu and proj == "gate_up_proj":+ name_g = f"{self._prefix}.gate_proj.{part}"+ name_u = f"{self._prefix}.up_proj.{part}"+ raw_g = self._shard_for(name_g).raw(name_g)+ raw_u = self._shard_for(name_u).raw(name_u)+ per = raw_g.size // self.num_experts+ rows = []+ for e in hot:+ rows.append(np.concatenate([+ raw_g[e * per:(e + 1) * per],+ raw_u[e * per:(e + 1) * per]]))+ else:+ name = f"{self._prefix}.{proj}.{part}"+ raw = self._shard_for(name).raw(name)+ per = raw.size // self.num_experts+ rows = [raw[e * per:(e + 1) * per] for e in hot]+ # concatenated numpy backing (page cache, not GPU)+ backing[(proj, part)] = np.concatenate(rows)+ self._hot_backing = backing+ self._hot_key = key+ self._hot_set = set(hot)+ # invalidate any materialized mx window content+ self._hot_weights = None++ def materialize_hot(self) -> None:+ """Materialize the numpy backing into mx arrays (GPU) for this+ layer. Called by the engine's before_layer_cb under the sliding+ window; the engine dematerializes layers outside the window."""+ if self._hot_weights is not None or self._hot_backing is None:+ return+ w = {}+ shape0 = self._bundle_shape+ n_e = len(self._hot_key)+ for (proj, part), buf in self._hot_backing.items():+ shape = shape0[(proj, part)]+ if part == "weight":+ # uint8 bytes -> u32 view, then [N, ...] stack+ arr = core.array(buf.view("<u4")).reshape(+ (n_e,) + shape[1:])+ else:+ # uint8 bytes -> bf16 bitcast view, then [N, ...] stack+ arr = core.array(buf.view("<u2")).view(+ core.bfloat16).reshape((n_e,) + shape[1:])+ w[(proj, part)] = arr+ self._hot_weights = w++ def dematerialize_hot(self) -> None:+ """Release the GPU copy; the numpy backing survives."""+ self._hot_weights = None++ def clear_hot_layer(self) -> None:+ self._hot_weights = None+ self._hot_backing = None+ self._hot_key = None+ self._hot_set = None++ # ---- forward ---------------------------------------------------------++ def _gather_gate_up(self, x, w, local, sorted_indices):+ """Up/gate projections over a stacked weight dict, fused or not."""+ if self._fuse_gu:+ x_gu = quant.gather_qmm(+ x, w[("gate_up_proj", "weight")],+ w[("gate_up_proj", "scales")],+ w[("gate_up_proj", "biases")], rhs_indices=local,+ transpose=True, group_size=self.group_size,+ bits=self.bits, mode=self.mode,+ sorted_indices=sorted_indices)+ x_gate, x_up = core.split(x_gu, 2, axis=-1)+ return x_up, x_gate+ x_up = quant.gather_qmm(+ x, w[("up_proj", "weight")], w[("up_proj", "scales")],+ w[("up_proj", "biases")], rhs_indices=local, transpose=True,+ group_size=self.group_size, bits=self.bits, mode=self.mode,+ sorted_indices=sorted_indices)+ x_gate = quant.gather_qmm(+ x, w[("gate_proj", "weight")], w[("gate_proj", "scales")],+ w[("gate_proj", "biases")], rhs_indices=local, transpose=True,+ group_size=self.group_size, bits=self.bits, mode=self.mode,+ sorted_indices=sorted_indices)+ return x_up, x_gate++ def __call__(self, x: core.array, indices: core.array | None = None) -> core.array:+ self._calls += 1+ # Record the router indices regardless of which path runs below, so+ # sync_actuals() can stage the next step's slots even after+ # fallbacks. In real-prerouter mode there are no router indices (the+ # staged set is the routing), so sync_actuals records the staged set.+ self._pending_indices = indices+ if self._calls % self.hot_update_interval == 0:+ self._refresh_hot_pins()++ # HOT-layer prefill path (multi-token chunks only): indices remapped+ # to the resident top-N stack; misses (~3%) contribute zero from the+ # fast gather (overflow row) and are computed separately below via+ # the exact per-expert path, then scatter-added. Numerically exact.+ if (self._hot_weights is not None and indices.size > 8+ and self._full_weights is None):+ with self._lock:+ flat = indices.reshape(-1).tolist()+ for e in set(int(v) for v in flat):+ self._hot_counts[e] = (+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)+ if self._staged_mode:+ k = indices.shape[-1]+ self._last_prefill_topk = flat[-k:]+ w = self._hot_weights+ pos = {e: i for i, e in enumerate(self._hot_key)}+ n_rows = len(self._hot_key)+ # normalize shapes: x may arrive [1, T, H] (batched) — flatten+ # to [T, H]; indices likewise [1, T, k] -> [T, k].+ x_flat = x.reshape(-1, x.shape[-1])+ ind_flat = indices.reshape(-1, indices.shape[-1])+ tok_n = ind_flat.shape[0]+ k_n = ind_flat.shape[-1]+ flat = ind_flat.reshape(-1).tolist()+ flat_idx = [int(v) for v in flat]+ g_list = []+ miss_rows = [] # flat positions needing exact add+ for i, e in enumerate(flat_idx):+ p = pos.get(e)+ if p is None:+ g_list.append(n_rows) # overflow zero row+ miss_rows.append(i)+ else:+ g_list.append(p)+ remapped = core.array(g_list, dtype=core.int32).reshape(+ ind_flat.shape)+ x3 = core.expand_dims(x_flat, (-2, -3))+ xs, local, inv_order = _gather_sort(x3, remapped)+ x_up, x_gate = self._gather_gate_up(xs, w, local, True)+ z = quant.gather_qmm(+ _swiglu(x_up, x_gate),+ w[("down_proj", "weight")], w[("down_proj", "scales")],+ w[("down_proj", "biases")], rhs_indices=local, transpose=True,+ group_size=self.group_size, bits=self.bits, mode=self.mode,+ sorted_indices=True)+ out = _scatter_unsort(z, inv_order)+ # Miss contribution: switch_mlp's caller applies scores after+ # us, so only RAW per-expert outputs [T, k, H] at miss slots are+ # needed. Reuse _moe_math (same kernel as the staged path) with+ # a single-expert "stack" built from the bundle.+ if miss_rows:+ miss_pairs = [(i // k_n, i % k_n, flat_idx[i])+ for i in miss_rows]+ by_expert = {}+ for t, kk, e in miss_pairs:+ by_expert.setdefault(e, []).append((t, kk))+ for e, tk in by_expert.items():+ b = self._build(e)+ rows = core.array([t for t, _ in tk], dtype=core.int32)+ xe = core.take(x_flat, rows, axis=0) # [m, H]+ xe2 = core.expand_dims(xe, (-2, -3)) # [m, 1, 1, H]+ # bundle tensors lack the expert dim -> add it once+ # (view-like reshape, no copy of the 4bit payload)+ def _e1(t):+ return t.reshape((1,) + t.shape)+ one = core.zeros((1,), dtype=core.int32)+ if self._fuse_gu:+ ggu = quant.gather_qmm(+ xe2, _e1(b[("gate_up_proj", "weight")]),+ _e1(b[("gate_up_proj", "scales")]),+ _e1(b[("gate_up_proj", "biases")]),+ rhs_indices=one, transpose=True,+ group_size=self.group_size, bits=self.bits,+ mode=self.mode, sorted_indices=True)+ gg, gu = core.split(ggu, 2, axis=-1)+ else:+ gu = quant.gather_qmm(+ xe2, _e1(b[("up_proj", "weight")]),+ _e1(b[("up_proj", "scales")]),+ _e1(b[("up_proj", "biases")]),+ rhs_indices=one, transpose=True,+ group_size=self.group_size, bits=self.bits,+ mode=self.mode, sorted_indices=True)+ gg = quant.gather_qmm(+ xe2, _e1(b[("gate_proj", "weight")]),+ _e1(b[("gate_proj", "scales")]),+ _e1(b[("gate_proj", "biases")]),+ rhs_indices=one, transpose=True,+ group_size=self.group_size, bits=self.bits,+ mode=self.mode, sorted_indices=True)+ ze = quant.gather_qmm(+ _swiglu(gu, gg),+ _e1(b[("down_proj", "weight")]),+ _e1(b[("down_proj", "scales")]),+ _e1(b[("down_proj", "biases")]),+ rhs_indices=one, transpose=True,+ group_size=self.group_size, bits=self.bits,+ mode=self.mode, sorted_indices=True+ ) # [m, 1, 1, H]+ ze = ze.squeeze(-2).squeeze(-2) # [m, H]+ flat_pos = core.array(+ [t * k_n + kk for t, kk in tk], dtype=core.int32)+ out = out.at[flat_pos].add(ze[:, None, :])+ out = out.reshape(*ind_flat.shape, *out.shape[-2:])+ out = out.squeeze(-2) # [T, k, H]+ if x.ndim == 3: # restore batch dim+ out = out.reshape(x.shape[0], tok_n, k_n, -1)+ return out++ # Whole-layer prefill path (multi-token chunks only): the full+ # stacked weights are already loaded (by the engine's+ # before_layer_cb load_full_layer / clear_full_layer), so the router+ # indices go straight into the sorted gather — same ops as the exact+ # path (bit-identical), no tolist / no bundles / no remap.+ if self._full_weights is not None and indices.size > 8:+ # Record usage so hot pins survive prefill (the whole-layer path+ # never touches _get_bundles, so without this _hot_counts stays+ # empty and PREFILL_PIN has nothing to select from). Also keep+ # the last token's top-k so the first decode step can be staged.+ with self._lock:+ flat = indices.reshape(-1).tolist()+ unique = set(int(v) for v in flat)+ for e in unique:+ self._hot_counts[e] = (+ self._hot_counts.get(e, 0.0) * self.hot_decay + 1.0)+ if self._staged_mode:+ k = indices.shape[-1]+ self._last_prefill_topk = flat[-k:]+ w = self._full_weights+ x = core.expand_dims(x, (-2, -3))+ x, local, inv_order = _gather_sort(x, indices)+ x_up, x_gate = self._gather_gate_up(x, w, local, True)+ z = quant.gather_qmm(+ _swiglu(x_up, x_gate),+ w[("down_proj", "weight")], w[("down_proj", "scales")],+ w[("down_proj", "biases")], rhs_indices=local, transpose=True,+ group_size=self.group_size, bits=self.bits, mode=self.mode,+ sorted_indices=True)+ # Matches the resident SwitchGLU contract: 4D [B, T, k, H] (the+ # caller's SparseMoeBlock sums over axis -2). NO batch restore+ # here — _gather_sort/_scatter_unsort already keep [B, T, k, H]+ # (verified against the deployment's whole-layer path, which+ # returns the same 4D shape).+ return _scatter_unsort(z, inv_order, indices.shape).squeeze(-2)++ # Staged decode path (prerouter-style, single-token): consume the+ # fixed staged slots; router indices never leave the GPU (expert->slot+ # via core.take). Missing experts map to the overflow zero slot and+ # their contribution is dropped. Falls back to the exact path only if+ # the slots are not ready yet.++ def _build_asm(bundles, slot_of_list, exp_key):+ slot_of = self._slot_table_cache.get(exp_key)+ if slot_of is None:+ slot_of = core.array(slot_of_list, dtype=core.int32)+ core.eval(slot_of)+ if len(self._slot_table_cache) > 8:+ self._slot_table_cache.pop(next(+ iter(self._slot_table_cache)))+ self._slot_table_cache[exp_key] = slot_of+ # Stack on the main thread (MLX lazy-op construction must not+ # race eval). Order matches the math fn's signature directly:+ # fused (w_gu, s_gu, b_gu, w_d, s_d, b_d); unfused+ # (w_u, s_u, b_u, w_g, s_g, b_g, w_d, s_d, b_d).+ wargs = []+ for proj in self._bundle_projs:+ for part in ("weight", "scales", "biases"):+ rows = [b[(proj, part)] for b in bundles]+ rows.extend([self._zero_slot[(proj, part)]] * (+ self._staged_n + 1 - len(rows)))+ wargs.append(core.stack(rows))+ return slot_of, tuple(wargs)++ # Real-prerouter routing: no router indices, route through the WHOLE+ # staged set (routing == prediction, no drops).+ if self._staged_replace and indices is None:+ self.wait_staged()+ st = self._staged_state+ if st is None:+ with self._lock:+ self._stats["staged_fallback"] += 1+ self._staged_at_use = None+ # First-step guard: zero contribution (nothing staged yet).+ return core.expand_dims(core.zeros_like(x), -2)+ bundles, slot_of_list, staged_exp = st+ self._staged_at_use = staged_exp+ self.last_used = list(staged_exp)+ exp_key = tuple(staged_exp)+ asm = self._staged_asm_cache.get(exp_key)+ if asm is None:+ slot_of, wargs = _build_asm(bundles, slot_of_list, exp_key)+ if len(self._staged_asm_cache) > 1:+ self._staged_asm_cache.pop(next(+ iter(self._staged_asm_cache)))+ self._staged_asm_cache[exp_key] = (slot_of, wargs)+ else:+ slot_of, wargs = asm+ k = len(staged_exp)+ local2d = core.arange(k, dtype=core.int32)[None, None, :]+ with self._lock:+ self._stats["staged_used"] += k+ return self._moe_math(x, *wargs, local2d)++ if self._staged_mode and indices is not None and indices.size == self._staged_trigger:+ self.wait_staged()+ st = self._staged_state+ if st is None:+ with self._lock:+ self._stats["staged_fallback"] += 1+ self._staged_at_use = None+ elif self._incr_mode and self._incr_tensors is not None:+ # Incremental stack: the persistent tensors were updated+ # in place at stage time (sticky slots); consume them+ # directly — no core.stack, no asm cache. Order matches the+ # math fn's signature exactly as _build_asm produces it.+ _, _, staged_exp = st+ self._staged_at_use = staged_exp+ wargs = tuple(+ self._incr_tensors[(proj, part)]+ for proj in self._bundle_projs+ for part in ("weight", "scales", "biases"))+ local2d = core.take(self._incr_slot_table, indices)+ with self._lock:+ self._stats["staged_used"] += indices.size+ return self._moe_math(x, *wargs, local2d)+ else:+ bundles, slot_of_list, staged_exp = st+ self._staged_at_use = staged_exp+ exp_key = tuple(staged_exp)+ asm = self._staged_asm_cache.get(exp_key)+ if asm is None:+ slot_of, wargs = _build_asm(+ bundles, slot_of_list, exp_key)+ if len(self._staged_asm_cache) > 1:+ self._staged_asm_cache.pop(next(+ iter(self._staged_asm_cache)))+ self._staged_asm_cache[exp_key] = (slot_of, wargs)+ else:+ slot_of, wargs = asm+ local2d = core.take(slot_of, indices)+ with self._lock:+ self._stats["staged_used"] += indices.size+ return self._moe_math(x, *wargs, local2d)++ # Exact decode path: resolve bundles for unique experts.+ _t0 = time.perf_counter()+ flat = indices.reshape(-1)+ unique = sorted(set(int(v) for v in flat.tolist()))+ with self._lock:+ self.last_used = unique+ _t1 = time.perf_counter()+ bundles = self._get_bundles(unique)+ _t2 = time.perf_counter()+ remap = {e: i for i, e in enumerate(unique)}+ local2d = core.array(+ [remap[e] for e in flat.tolist()]+ ).reshape(indices.shape)+ _t3 = time.perf_counter()++ vals = list(bundles.values())+ wargs = []+ for proj in self._bundle_projs:+ for part in ("weight", "scales", "biases"):+ wargs.append(core.stack([b[(proj, part)] for b in vals]))++ orig_x = x+ x = core.expand_dims(x, (-2, -3))+ do_sort = local2d.size >= 64+ inv_order = None+ if do_sort:+ if self._use_compile:+ x_s, local, inv_order = _gather_sort(x, local2d)+ return self._moe_math_sorted(+ x_s, *wargs, local, inv_order, indices.shape)+ x, local, inv_order = _gather_sort(x, local2d)+ elif self._use_compile:+ return self._moe_math(orig_x, *wargs, local2d)+ else:+ local = local2d++ if self._fuse_gu:+ w_gu, s_gu, b_gu = wargs[0], wargs[1], wargs[2]+ w_d, s_d, b_d = wargs[3], wargs[4], wargs[5]+ x_gu = quant.gather_qmm(+ x, w_gu, s_gu, b_gu, rhs_indices=local,+ transpose=True, group_size=self.group_size, bits=self.bits,+ mode=self.mode, sorted_indices=do_sort)+ x_gate, x_up = core.split(x_gu, 2, axis=-1)+ else:+ # wargs order matches the math contract: up first, then gate,+ # then down (see spec.bundle_projs).+ w_u, s_u, b_u = wargs[0], wargs[1], wargs[2]+ w_g, s_g, b_g = wargs[3], wargs[4], wargs[5]+ w_d, s_d, b_d = wargs[6], wargs[7], wargs[8]+ x_up = quant.gather_qmm(+ x, w_u, s_u, b_u, rhs_indices=local,+ transpose=True, group_size=self.group_size, bits=self.bits,+ mode=self.mode, sorted_indices=do_sort)+ x_gate = quant.gather_qmm(+ x, w_g, s_g, b_g, rhs_indices=local,+ transpose=True, group_size=self.group_size, bits=self.bits,+ mode=self.mode, sorted_indices=do_sort)+ z = quant.gather_qmm(+ _swiglu(x_up, x_gate), w_d, s_d, b_d, rhs_indices=local,+ transpose=True, group_size=self.group_size, bits=self.bits,+ mode=self.mode, sorted_indices=do_sort)+ if do_sort:+ z = _scatter_unsort(z, inv_order, indices.shape)+ return z.squeeze(-2)++ # ---- lifecycle --------------------------------------------------------++ def reset(self):+ """Per-request reset: drop staged/prefill working state so the next+ request starts clean (the shared LRU, hot counts, and pools survive+ across requests — they are the cross-request memory)."""+ self._pending_indices = None+ self._staged_at_use = None+ self._last_prefill_topk = None+ self._staged_state = None+ self._staged_state_next = None+ with self._staging_lock:+ self._staged_pending = None+ self.last_used = []+ self._full_weights = None++ def close(self):+ """Release thread pools (idempotent)."""+ for pool in (self._pool, self._prefetch_pool):+ try:+ pool.shutdown(wait=False, cancel_futures=True)+ except Exception:+ passdiff --git a/src/edge0/streaming/mmap.py b/src/edge0/streaming/mmap.pynew file mode 100644index 0000000..3ea9441--- /dev/null+++ b/src/edge0/streaming/mmap.py@@ -0,0 +1,63 @@+"""Zero-copy byte-range access into safetensors shards.++A single expert is a contiguous byte slice of one stacked+[num_experts, ...] tensor; the streaming layer reads those slices straight+out of an mmap with no full-tensor materialization.+"""++from __future__ import annotations++import json+import mmap+import struct++import numpy as np+++class SafetensorsMmap:+ """One safetensors shard, opened as a byte-range mmap.++ ``entries[name]`` holds ``{"offset", "size", "dtype", "shape"}``;+ ``raw(name)`` returns a zero-copy uint8 view of the tensor's bytes.+ """++ def __init__(self, path: str):+ self.path = path+ self._file = open(path, "rb")+ self._mm = mmap.mmap(self._file.fileno(), 0, access=mmap.ACCESS_READ)+ header_len = struct.unpack("<Q", self._mm[:8])[0]+ header = json.loads(self._mm[8:8 + header_len])+ self.payload_base = 8 + header_len+ self.entries = {}+ for name, meta in header.items():+ if name == "__metadata__":+ continue+ begin, end = meta["data_offsets"]+ self.entries[name] = {+ "offset": self.payload_base + begin,+ "size": end - begin,+ "dtype": meta["dtype"],+ "shape": tuple(meta["shape"]),+ }++ def raw(self, name: str) -> np.ndarray:+ e = self.entries[name]+ return np.frombuffer(+ self._mm, dtype=np.uint8, count=e["size"], offset=e["offset"]+ )++ def close(self):+ self._mm.close()+ self._file.close()+++def bf16_bits_to_f32(data: np.ndarray) -> np.ndarray:+ """Raw BF16 bytes -> float32 array (bit pattern shifted into fp32)."""+ u16 = data.view("<u2")+ return (u16.astype(np.uint32) << 16).view(np.float32)+++def u32_view(data: np.ndarray, shape) -> np.ndarray:+ """uint8 bytes -> uint32 view reshaped to ``shape`` (quantized payloads+ are stored as packed uint32 words)."""+ return data.view("<u4").reshape(shape)diff --git a/src/edge0/streaming/options.py b/src/edge0/streaming/options.pynew file mode 100644index 0000000..115130e--- /dev/null+++ b/src/edge0/streaming/options.py@@ -0,0 +1,134 @@+"""Streaming layer options.++One ``LayerOptions`` instance describes the streaming behavior of one MoE+layer (all layers usually share one instance). This is the typed+replacement for the deployment's env-var knobs; profiles (see+``edge0.models``) pick the option sets for a given tier.++Defaults reproduce the deployment's production behavior for the edge0-35b+K=4 tier (staged decode on, sync fills, hot pins on prefill).+"""++from __future__ import annotations++from dataclasses import dataclass, field+++@dataclass+class LayerOptions:+ """Options for one streaming MoE layer.++ Attributes:+ staged: enable fixed-slot staged decode (routing through staged+ slots with zero per-layer host syncs).+ staged_replace: staged set REPLACES the router (routing set ==+ staged set; used with a prerouter head — no drops).+ staged_n: number of staged slots.+ staged_trigger: router top-k that activates the staged path+ (usually the routed top_k).+ staged_sync: fill staged slots synchronously at the step boundary+ (as opposed to purely async double-buffered fills).+ asm_cache: cache assembled (slot table, stacked-wargs) per staged+ expert set so repeated sets don't rebuild graph nodes.+ incr_stack: incremental sticky-slot stack (in-place row writes+ instead of re-stacking; sync staged mode only).+ incr_writeback: write evicted bundles back into the shared LRU+ (dedups the staged set from the LRU while staged).+ history_prefetch: prefetch the union of recent actual expert sets.+ hot_per_layer: resident hot-expert pins per layer (0 disables).+ hot_update_interval: refresh hot pins every N forward calls.+ hot_decay: decay factor for hot-expert usage counts.+ pin_bonus: extra score for prerouter-predicted experts when+ selecting hot pins.+ cache_slots: shared LRU capacity across all layers.+ prefetch_cap: prefetch buffer capacity.+ load_threads: threads for on-demand expert builds.+ prefetch_threads: threads for eager prefetch builds.+ use_compile: wrap the MoE math in ``mx.compile``.+ top_k: override the resident block's routed top-k (both prefill+ and decode); None leaves the model config value.+ full_layer_prefill: whole-layer load-drop prefill (E3b).+ prefill_full_layers: number of LEADING layers that get whole-layer+ loads (N*~310MB of page cache for qwen); 0 = every layer.+ prefill_hot: resident hot-stack size used during prefill+ (0 disables the hot-stack path).+ """++ staged: bool = False+ staged_replace: bool = False+ staged_n: int = 8+ staged_trigger: int = 8+ staged_sync: bool = True+ asm_cache: bool = True+ incr_stack: bool = False+ incr_writeback: bool = False+ history_prefetch: bool = True+ hot_per_layer: int = 0+ hot_update_interval: int = 4+ hot_decay: float = 0.75+ pin_bonus: float = 2.0+ cache_slots: int = 64+ prefetch_cap: int = 48+ load_threads: int = 8+ prefetch_threads: int = 4+ use_compile: bool = True+ top_k: int | None = None+ full_layer_prefill: bool = False+ prefill_full_layers: int = 0+ prefill_hot: int = 0++ # ---- presets ----------------------------------------------------------++ @classmethod+ def staged_k4(cls, **overrides) -> "LayerOptions":+ """edge0-35b K=4 tier: staged decode + hot pins + whole-layer+ prefill (production profile).++ ``staged_replace`` stays False: the trained prerouter head supplies+ the routing (the MoE block routes via the prerouter logits), so the+ staged set == the routing set exactly — the slot table maps without+ drops."""+ return cls(+ staged=True, staged_replace=False, staged_n=4,+ staged_trigger=4, staged_sync=True, history_prefetch=True,+ hot_per_layer=32, hot_update_interval=4, hot_decay=0.75,+ cache_slots=64, prefetch_cap=48, load_threads=8,+ prefetch_threads=4, use_compile=True, top_k=4,+ full_layer_prefill=True, prefill_full_layers=12,+ prefill_hot=32, **overrides)++ @classmethod+ def staged_k8(cls, **overrides) -> "LayerOptions":+ """edge0-10b tier: staged decode with K=8 (native routing width),+ no hot pins, E3b whole-layer prefill."""+ return cls(+ staged=True, staged_replace=False, staged_n=8,+ staged_trigger=8, staged_sync=True, history_prefetch=True,+ hot_per_layer=0, cache_slots=64, prefetch_cap=48,+ load_threads=8, prefetch_threads=4, use_compile=True, top_k=8,+ full_layer_prefill=True, prefill_hot=0, **overrides)++ @classmethod+ def prod_k8(cls, **overrides) -> "LayerOptions":+ """edge0-10b tier: deployment production profile.++ Mirrors start_server.sh on the ling v7 deploy: STAGED_DECODE=0+ (staged decode is OFF — the deployment verified staged decode on+ ling degrades output, see the comment in start_server.sh),+ STAGED_SYNC=0, STAGED_SLOTS=8, EXPERT_CACHE_SLOTS=64,+ MLX_COMPILE_MOE=1, PREFILL_FULL_LAYER=1. The hybrid prerouter+ still stages each next token's expert set into the shared LRU+ (engine.py: stage_all at the step boundary AND right after+ prefill), and the vendored MoE blocks re-select from the cached+ prerouter logits."""+ return cls(+ staged=False, staged_replace=False, staged_n=8,+ staged_trigger=8, staged_sync=False, history_prefetch=False,+ hot_per_layer=0, cache_slots=64, prefetch_cap=48,+ load_threads=8, prefetch_threads=4, use_compile=True, top_k=8,+ full_layer_prefill=True, prefill_hot=0, **overrides)++ def for_layer(self, layer_idx: int) -> "LayerOptions":+ """Per-layer copy (staged slots are per-layer state, but options+ are shared; this hook exists for future per-layer overrides)."""+ return selfdiff --git a/tests/test_moe_spec.py b/tests/test_moe_spec.pynew file mode 100644index 0000000..b3dd18f--- /dev/null+++ b/tests/test_moe_spec.py@@ -0,0 +1,98 @@+"""MoESpec path resolution and key templating tests."""++from __future__ import annotations++import pytest++from edge0.moe.spec import (MoESpec, QuantSpec, RouterKind, WeightLayout)+++class _FakeLayer:+ def __init__(self, mlp=None):+ self.mlp = mlp+++class _FakeSwitchMLP:+ def __init__(self):+ self.name = "switch"+++class _FakeBlock:+ def __init__(self):+ self.switch_mlp = _FakeSwitchMLP()+++class _FakeModel:+ def __init__(self, n=4):+ self.language_model = type("LM", (), {})()+ self.language_model.model = type("M", (), {})()+ self.language_model.model.layers = [+ _FakeLayer(_FakeBlock()) for _ in range(n)+ ]+ # ling-style plain module list (no attribute access)+ self.layers = [_FakeLayer(_FakeBlock()) for _ in range(n)]+++def _spec(**kw):+ base = dict(+ num_experts=8, top_k=2, intermediate_size=64,+ key_template="language_model.model.layers.{layer}.mlp.switch_mlp",+ block_path="language_model.model.layers.{layer}.mlp.switch_mlp",+ )+ base.update(kw)+ return MoESpec(**base)+++def test_keys_templating():+ s = _spec()+ assert s.keys(3, "gate_proj", "weight") == (+ "language_model.model.layers.3.mlp.switch_mlp.gate_proj.weight")+ assert s.keys(12, "down_proj", "biases") == (+ "language_model.model.layers.12.mlp.switch_mlp.down_proj.biases")+++def test_block_of_digit_segments():+ s = _spec()+ m = _FakeModel()+ block = s.block_of(m, 2)+ assert block.name == "switch"+ assert s.block_of(m, 0) is m.language_model.model.layers[0].mlp.switch_mlp+++def test_layer_of_defaults_from_block_path():+ s = _spec()+ m = _FakeModel()+ layer = s.layer_of(m, 1)+ assert layer is m.language_model.model.layers[1]+++def test_layer_of_plain_list_path():+ s = _spec(block_path="layers.{layer}.mlp",+ layer_path="layers.{layer}",+ key_template="model.layers.{layer}.mlp.experts")+ m = _FakeModel()+ assert s.block_of(m, 3) is m.layers[3].mlp+ assert s.layer_of(m, 2) is m.layers[2]+++def test_bundle_projs_layouts():+ # Stack order matches the math signature directly: up, gate, down+ # (_swiglu(up, gate)) — same convention as the original+ # streaming_experts_qwen35.py ("up_proj", "gate_proj", "down_proj").+ sep = _spec()+ assert sep.bundle_projs == ("up_proj", "gate_proj", "down_proj")+ assert sep.fuse_gu is False+ fused = _spec(layout=WeightLayout.FUSED_GATE_UP)+ assert fused.bundle_projs == ("gate_up_proj", "down_proj")+ assert fused.fuse_gu is True+++def test_quant_defaults():+ assert QuantSpec().bits == 4+ assert QuantSpec().group_size == 64+ assert QuantSpec().mode == "affine"+++def test_router_kinds():+ assert RouterKind.SOFTMAX_TOPK.value == "softmax_topk"+ assert RouterKind.SIGMOID_GROUP.value == "sigmoid_group"diff --git a/tests/test_registry.py b/tests/test_registry.pynew file mode 100644index 0000000..adf9998--- /dev/null+++ b/tests/test_registry.py@@ -0,0 +1,97 @@+"""Registry / AutoConfig / AutoModel resolution tests (no GPU needed)."""++from __future__ import annotations++import pytest++import edge0.models # noqa: F401 (populates MODEL_REGISTRY on import)+from edge0 import AutoConfig+from edge0.registry import MODEL_REGISTRY, TYPE_ALIASES+++def test_both_tiers_registered():+ assert set(MODEL_REGISTRY) == {"edge0-35b", "edge0-10b"}+++def test_type_aliases_resolve():+ assert TYPE_ALIASES["qwen3_5_moe"] == "edge0-35b"+ assert TYPE_ALIASES["qwen3_5_moe_text"] == "edge0-35b"+ assert TYPE_ALIASES["bailing_hybrid"] == "edge0-10b"+ assert TYPE_ALIASES["bailing_moe_linear"] == "edge0-10b"++[email protected]("name", ["edge0-35b", "edge0-10b"])+def test_auto_config_defaults(name):+ cfg = AutoConfig.from_pretrained(name=name)+ assert cfg.name == name+ assert cfg.moe_spec.num_experts > 0+ assert cfg.moe_spec.top_k in (4, 8)+ # qwen tier defaults to staged decode; the ling tier mirrors the+ # deployment production profile (STAGED_DECODE=0 — staged decode on+ # ling degrades output, see start_server.sh) with the prerouter+ # still staging each next token's expert set.+ assert cfg.options.staged is (name == "edge0-35b")+ assert cfg.prerouter is not None+ assert cfg.prerouter.weights_file.endswith(".safetensors")+ assert cfg.prerouter_top_k == cfg.moe_spec.top_k+++def test_qwen35_profile():+ cfg = AutoConfig.from_pretrained(name="edge0-35b")+ assert cfg.moe_spec.num_experts == 256+ assert cfg.moe_spec.top_k == 4+ assert cfg.moe_spec.intermediate_size == 512+ assert cfg.moe_spec.norm_topk_prob is True+ assert cfg.moe_spec.shared_experts == 1+ assert cfg.moe_spec.quant.bits == 4+ assert cfg.moe_spec.quant.group_size == 64+ assert cfg.moe_spec.layout.value == "separate"+ assert "language_model.model.layers" in cfg.moe_spec.key_template+ assert cfg.options.staged_n == 4+ assert cfg.options.prefill_full_layers == 12+ assert cfg.options.hot_per_layer == 32+ assert cfg.prerouter.start_layer == 7+ assert cfg.prerouter.hidden == 512+ assert cfg.prerouter.dtype == "fp16"+ assert cfg.prerouter.feature_topk == "executed"+ assert cfg.gen.temperature == 0.6+ assert 248046 in cfg.gen.eos_ids+ assert cfg.port == 8085+++def test_ling10b_profile():+ cfg = AutoConfig.from_pretrained(name="edge0-10b")+ assert cfg.moe_spec.num_experts == 128+ assert cfg.moe_spec.top_k == 8+ assert cfg.moe_spec.router.value == "sigmoid_group"+ assert cfg.moe_spec.routed_scaling == 2.5+ assert cfg.moe_spec.n_group == 8+ assert cfg.moe_spec.topk_group == 4+ assert cfg.options.staged_n == 8+ assert cfg.prerouter.start_layer == 1+ assert cfg.prerouter.owners == tuple(range(1, 23))+ assert cfg.prerouter.patch_call is False+ assert 156895 in cfg.gen.eos_ids+ assert cfg.port == 8083+++def test_override_and_reject():+ cfg = AutoConfig.from_pretrained(name="edge0-10b", prerouter=None,+ lora="", prerouter_top_k=0)+ assert cfg.prerouter is None+ assert cfg.lora == ""+ assert cfg.prerouter_top_k == 0+ with pytest.raises(TypeError, match="unknown"):+ AutoConfig.from_pretrained(name="edge0-10b", bogus_field=1)+++def test_unknown_name_lists_registry():+ with pytest.raises(KeyError, match="edge0-35b"):+ AutoConfig.from_pretrained(name="edge0-99b")+++def test_adapter_module_exposes_config():+ for name, mod in MODEL_REGISTRY.items():+ assert hasattr(mod, "Config")+ assert hasattr(mod, "build_model")+ assert hasattr(mod, "build_engine")diff --git a/tests/test_sampling.py b/tests/test_sampling.pynew file mode 100644index 0000000..965c01c--- /dev/null+++ b/tests/test_sampling.py@@ -0,0 +1,64 @@+"""Sampler unit tests: shape-independent draws and truncation behavior."""++from __future__ import annotations++import mlx.core as mx+import pytest++from edge0.sampling import _mask_logits, sample+++def test_mask_identity_for_sane_kwargs():+ logits = mx.arange(10, dtype=mx.float32)+ out = _mask_logits(logits, temperature=1.0, top_k=None, top_p=None)+ assert mx.allclose(out, logits).item()+++def test_mask_topk_keeps_only_top_k():+ logits = mx.arange(10, dtype=mx.float32) # 0..9+ out = _mask_logits(logits, temperature=1.0, top_k=3, top_p=None)+ vals = mx.sort(out, axis=-1)[-3:].tolist()+ assert vals == [7.0, 8.0, 9.0]+ assert int(mx.sum(mx.isfinite(out)).item()) == 3+++def test_sample_returns_int_and_in_range():+ logits = mx.ones((5,), dtype=mx.float32)+ for _ in range(5):+ t = sample(logits, temperature=1.0)+ assert isinstance(t, int)+ assert 0 <= t < 5+++def test_sample_seed_deterministic():+ logits = mx.arange(8, dtype=mx.float32) * 2.0+ a = sample(logits, temperature=0.5, seed=42)+ b = sample(logits, temperature=0.5, seed=42)+ assert a == b+++def test_repetition_penalty_changes_distribution():+ logits = mx.array([10.0, 9.0, 0.0], dtype=mx.float32)+ # history token 0 is penalized -> token 1 becomes likelier+ t = sample(logits, temperature=1.0, repetition_penalty=2.0,+ history=(0,))+ assert isinstance(t, int)+ # with penalty 2x on a +10 logit: 5 vs 9 -> argmax flips to 1+ out = _mask_logits(logits, 1.0, None, None)+ assert out.dtype == mx.float32+++def test_sample_temperature_zero_is_argmax():+ logits = mx.array([1.0, 5.0, 3.0], dtype=mx.float32)+ # temperature <= 0 is deterministic decode (HF / llama.cpp semantics):+ # greedy — the draw must ALWAYS be the argmax, whatever the seed.+ for seed in range(20):+ assert sample(logits, temperature=0.0, seed=seed) == 1+++def test_sample_temperature_zero_greedy_with_topk():+ # top-k truncation still applies before the greedy argmax+ logits = mx.array([1.0, 5.0, 3.0, 4.0, 2.0], dtype=mx.float32)+ # top-2 = {5.0 (idx 1), 4.0 (idx 3)} -> greedy picks idx 1+ for seed in range(5):+ assert sample(logits, temperature=0.0, top_k=2, seed=seed) == 1diff --git a/tests/test_server.py b/tests/test_server.pynew file mode 100644index 0000000..7c7a373--- /dev/null+++ b/tests/test_server.py@@ -0,0 +1,317 @@+"""Server-layer tests: request parsing, chat session, SSE framing and the+stdlib HTTP transport, all against a fake engine (no real checkpoints).++Covers the OpenAI-compatible surface the deployment servers expose:+/healthz, /v1/models, POST /v1/chat/completions (non-streaming and SSE+streaming), and the /v1/completions rejection path.+"""++from __future__ import annotations++import json+import threading+import time+from http.server import ThreadingHTTPServer+from types import SimpleNamespace+from urllib import request as urlrequest++import pytest++from edge0.config import GenerationConfig+from edge0.server.app import (+ _chat_once,+ _chat_stream,+ _StdlibHandler,+ build_app_handlers,+ run_stdlib,+)+from edge0.server.chat import (+ ChatMessage,+ ChatRequest,+ ChatSession,+ QueueServer,+ decode_tokens,+ parse_chat_request,+ sse_format,+)+++# ---- fake engine ----------------------------------------------------------+++class FakeTok:+ """Minimal tokenizer stub (transformers-like surface)."""++ def __init__(self, ids=(7, 8, 9)):+ self.ids = list(ids)+ self.bos_token_id = 0++ def apply_chat_template(self, messages, tokenize=False,+ add_generation_prompt=True,+ enable_thinking=False):+ text = "<tpl>" + json.dumps(messages)+ if add_generation_prompt:+ text += "<|im_start|>assistant\n"+ return text++ def encode(self, text):+ return self.ids++ def decode(self, tokens):+ return "".join(f"T{t}" for t in tokens)+++class PlainTok(FakeTok):+ """Tokenizer without apply_chat_template (stdlib fallback path)."""++ def apply_chat_template(self, *a, **k): # pragma: no cover - unused+ raise AttributeError+++class FakeEngine:+ """Duck-typed engine: enough of Edge0Engine for the server layer."""++ name = "fake"++ def __init__(self, tok=None):+ self._tok = tok or FakeTok()+ self.pos = 3+ self.cfg = SimpleNamespace(+ gen=GenerationConfig(temperature=0.7, top_p=0.95))+ self.generated = []++ def generate(self, ids, gen_config=None, on_token=None, **kw):+ toks = [11, 12, 13]+ self.generated.append((list(ids), gen_config))+ if on_token is not None:+ for t in toks:+ on_token(t)+ return toks++ def stats(self):+ return {"ok": 1}+++def _req(**kw) -> ChatRequest:+ payload = {+ "model": "fake",+ "messages": [{"role": "user", "content": "hi"}],+ **kw,+ }+ return parse_chat_request(payload)+++# ---- parse_chat_request ---------------------------------------------------+++def test_parse_basic_defaults():+ req = _req()+ assert req.model == "fake"+ assert len(req.messages) == 1+ assert req.messages[0].role == "user"+ assert req.messages[0].content == "hi"+ assert req.temperature is None+ assert req.stream is False+++def test_parse_multipart_content_joins_text():+ req = _req(messages=[{+ "role": "user",+ "content": [{"type": "text", "text": "a"},+ {"type": "image_url", "image_url": {"url": "x"}},+ {"type": "text", "text": "b"}],+ }])+ assert req.messages[0].content == "ab"+++def test_parse_stream_flag_and_sampling():+ req = _req(temperature=0.3, top_p=0.9, top_k=32, max_tokens=16,+ stream=True)+ assert req.temperature == 0.3+ assert req.top_p == 0.9+ assert req.top_k == 32+ assert req.max_tokens == 16+ assert req.stream is True+++# ---- sse / decode ---------------------------------------------------------+++def test_sse_format():+ line = sse_format({"id": "1", "x": "中文"})+ assert line.startswith("data: ")+ assert line.endswith("\n\n")+ assert json.loads(line[6:-2]) == {"id": "1", "x": "中文"}+++def test_decode_tokens():+ eng = FakeEngine()+ assert decode_tokens(eng, [1, 2]) == "T1T2"+ eng._tok = None+ assert decode_tokens(eng, [1]) == ""+++# ---- ChatSession ----------------------------------------------------------+++def test_chat_session_prompt_ids_uses_template():+ eng = FakeEngine()+ sess = ChatSession(eng, _req())+ ids = sess.prompt_ids()+ assert ids == [7, 8, 9]+ # The template was called with enable_thinking=False (serve.py parity).+ text = eng._tok.apply_chat_template([{"role": "user", "content": "hi"}])+ assert "<|im_start|>assistant" in text+++def test_chat_session_prompt_ids_fallback_text():+ eng = FakeEngine(tok=PlainTok())+ sess = ChatSession(eng, _req())+ ids = sess.prompt_ids()+ assert ids == [7, 8, 9]+++def test_chat_session_fallback_text_format():+ eng = FakeEngine(tok=PlainTok())+ req = parse_chat_request({"messages": [+ {"role": "system", "content": "sys"},+ {"role": "user", "content": "q"},+ {"role": "assistant", "content": "a"},+ ]})+ sess = ChatSession(eng, req)+ text = sess._chat_text()+ assert "<|im_start|>system\nsys<|im_end|>\n" in text+ assert "<|im_start|>user\nq<|im_end|>\n" in text+ assert "<|im_start|>assistant\na<|im_end|>\n" in text+ assert text.endswith("<|im_start|>assistant\n")+++def test_gen_config_overrides_request_fields():+ eng = FakeEngine()+ sess = ChatSession(eng, _req(temperature=0.1, max_tokens=5))+ gen = sess.gen_config()+ assert gen.temperature == 0.1+ assert gen.max_new_tokens == 5+ assert gen.top_p == 0.95 # engine default preserved+++def test_chat_session_run_usage_and_callback():+ eng = FakeEngine()+ seen = []+ sess = ChatSession(eng, _req())+ tokens, meta = sess.run(on_token=seen.append)+ assert tokens == [11, 12, 13]+ assert seen == [11, 12, 13]+ assert meta["usage"] == {+ "prompt_tokens": 3,+ "completion_tokens": 3,+ "total_tokens": 6,+ }+ assert "wall_s" in meta+++# ---- QueueServer ----------------------------------------------------------+++def test_queue_server_health():+ eng = FakeEngine()+ srv = QueueServer(eng)+ h = srv.health()+ assert h["status"] == "ok"+ assert h["model"] == "fake"+ assert h["pos"] == 3+++def test_queue_server_chat_serializes():+ eng = FakeEngine()+ srv = QueueServer(eng)+ out = srv.chat(_req())+ assert out["usage"]["completion_tokens"] == 3+ assert len(eng.generated) == 1+ gen = eng.generated[0][1]+ assert gen.temperature == 0.7+++# ---- handlers -------------------------------------------------------------+++def test_handlers_healthz_and_models():+ srv = QueueServer(FakeEngine())+ handlers = build_app_handlers(srv)+ assert handlers["GET /healthz"]()["status"] == "ok"+ models = handlers["GET /v1/models"]()+ assert models["object"] == "list"+ assert models["data"][0]["id"] == "fake"+++def test_handlers_completions_rejected():+ srv = QueueServer(FakeEngine())+ handlers = build_app_handlers(srv)+ body, status = handlers["POST /v1/completions"]({})+ assert status == 400+++def test_chat_once_response_shape():+ srv = QueueServer(FakeEngine())+ out = _chat_once(srv, {"messages": [{"role": "user", "content": "hi"}]})+ assert out["object"] == "chat.completion"+ assert out["model"] == "fake"+ ch = out["choices"][0]+ assert ch["message"]["role"] == "assistant"+ assert ch["message"]["content"] == "T11T12T13"+ assert ch["finish_reason"] == "stop"+ assert out["usage"]["completion_tokens"] == 3+++def test_chat_stream_events_sequence():+ srv = QueueServer(FakeEngine())+ raw = _chat_stream(srv, {+ "messages": [{"role": "user", "content": "hi"}], "stream": True,+ })+ body = raw if isinstance(raw, bytes) else raw.get_data()+ events = [e for e in body.decode("utf-8").split("\n\n") if e]+ chunks = [json.loads(e[6:]) for e in events[:-1]] # drop [DONE]+ deltas = [c["choices"][0]["delta"]["content"] for c in chunks]+ assert deltas == ["T11", "T12", "T13"]+ assert chunks[-1]["choices"][0]["finish_reason"] == "stop"+ assert events[-1] == "data: [DONE]"+++# ---- stdlib HTTP transport ------------------------------------------------+++def test_stdlib_http_end_to_end():+ eng = FakeEngine()+ srv = QueueServer(eng, model_name="fake")+ handler = type("Edge0Handler", (_StdlibHandler,), {"server_q": srv})+ httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler)+ port = httpd.server_address[1]+ t = threading.Thread(target=httpd.serve_forever, daemon=True)+ t.start()+ base = f"http://127.0.0.1:{port}"+ try:+ with urlrequest.urlopen(f"{base}/healthz", timeout=10) as r:+ assert r.status == 200+ assert json.loads(r.read())["status"] == "ok"+ with urlrequest.urlopen(f"{base}/v1/models", timeout=10) as r:+ assert json.loads(r.read())["data"][0]["id"] == "fake"+ payload = json.dumps({+ "model": "fake",+ "messages": [{"role": "user", "content": "hi"}],+ "max_tokens": 8,+ }).encode("utf-8")+ req = urlrequest.Request(f"{base}/v1/chat/completions",+ data=payload,+ headers={"Content-Type": "application/json"})+ with urlrequest.urlopen(req, timeout=10) as r:+ assert r.status == 200+ out = json.loads(r.read())+ assert out["choices"][0]["message"]["content"] == "T11T12T13"+ # unknown route -> 404+ with pytest.raises(Exception) as exc:+ urlrequest.urlopen(f"{base}/nope", timeout=10)+ assert getattr(exc.value, "code", None) == 404+ finally:+ httpd.shutdown()+ httpd.server_close()+ t.join(timeout=5)diff --git a/tests/test_streaming_math.py b/tests/test_streaming_math.pynew file mode 100644index 0000000..e77154e--- /dev/null+++ b/tests/test_streaming_math.py@@ -0,0 +1,301 @@+"""StreamingSwitchGLU path math tests: every streaming path must produce+outputs identical to the exact per-expert path (same gather_qmm kernel),+and the exact path must match a plain fp16 reference implementation.++No real checkpoints needed: small random 4-bit weights are written to a+temp safetensors shard with the exact layout the checkpoint uses+(gate_proj / up_proj / down_proj stacked along the expert axis).+"""++from __future__ import annotations++import numpy as np+import pytest+import mlx.core as mx+import mlx.nn as nn+from safetensors.numpy import save_file++from edge0.moe.spec import MoESpec, QuantSpec, WeightLayout+from edge0.streaming.cache import SharedExpertCache+from edge0.streaming.layer import StreamingSwitchGLU+from edge0.streaming.mmap import SafetensorsMmap+from edge0.streaming.options import LayerOptions++N_EXPERTS = 8+DIM = 64+INTER = 128+++def _write_shard(path, fuse_gu: bool, seed: int = 0):+ """Random fp16 weights -> 4-bit shard; returns the fp16 originals.++ Layout matches the checkpoint convention [out, in/8] packed u32:+ gate/up project hidden (DIM) to intermediate (INTER); down projects+ INTER back to DIM — so down rows run along DIM with INTER as the+ packed axis.+ """+ mx.random.seed(seed)+ w = {}+ w["gate_proj"] = mx.random.normal((N_EXPERTS, INTER, DIM)).astype(mx.float16)+ w["up_proj"] = mx.random.normal((N_EXPERTS, INTER, DIM)).astype(mx.float16)+ w["down_proj"] = mx.random.normal((N_EXPERTS, DIM, INTER)).astype(mx.float16)++ def _parts(proj):+ w16 = w[proj]+ # transpose to [E, out, in] -> quantize per-expert rows along last+ # axis; scales are stored as bf16 bit patterns (u16), weights as+ # packed u32.+ q, scales, biases = mx.quantize(w16, group_size=64, bits=4)+ # mx.quantize returns fp16 scales; the checkpoint stores bf16 bit+ # patterns — convert numerically, then bitcast to u16.+ return {+ "weight": np.asarray(q), # uint32+ "scales": np.asarray(scales.astype(mx.bfloat16).view(mx.uint16)),+ "biases": np.asarray(biases.astype(mx.bfloat16).view(mx.uint16)),+ }++ tensors = {}+ for proj in ("gate_proj", "up_proj", "down_proj"):+ for part, arr in _parts(proj).items():+ tensors[f"layers.0.mlp.switch_mlp.{proj}.{part}"] = arr+ save_file(tensors, str(path))+ return {k: np.asarray(v) for k, v in w.items()}+++def _spec(fuse_gu: bool) -> MoESpec:+ return MoESpec(+ num_experts=N_EXPERTS, top_k=4, intermediate_size=INTER,+ quant=QuantSpec(bits=4, group_size=64),+ layout=(WeightLayout.FUSED_GATE_UP if fuse_gu+ else WeightLayout.SEPARATE),+ key_template="layers.{layer}.mlp.switch_mlp",+ block_path="layers.{layer}.mlp.switch_mlp",+ )+++def _options(**kw) -> LayerOptions:+ base = dict(staged=False, staged_n=4, staged_trigger=4,+ staged_sync=True, load_threads=2, prefetch_threads=1)+ base.update(kw)+ return LayerOptions(**base)++[email protected](scope="module", params=[False, True],+ ids=["separate", "fused_gate_up"])+def layer(request, tmp_path_factory):+ fuse = request.param+ path = tmp_path_factory.mktemp("shard") / "w.safetensors"+ _write_shard(path, fuse)+ mm = SafetensorsMmap(str(path))+ lay = StreamingSwitchGLU(+ [mm], 0, _spec(fuse), _options(),+ shared_cache=SharedExpertCache(64))+ return lay, mm, fuse+++def ref_moe(x, inds, shard: SafetensorsMmap, fuse_gu: bool):+ """Bit-level reference: dequantize the SAME 4-bit payloads the+ streaming paths consume, then compute the GLU in float32. This pins+ the file layout + dequant math; kernel rounding aside it must match+ every streaming path exactly."""+ xf = x.astype(mx.float32)+ wcache = {}++ def w(e, proj):+ key = (e, proj)+ if key not in wcache:+ name = f"layers.0.mlp.switch_mlp.{proj}.weight"+ raw = shard.raw(name)+ per = raw.size // N_EXPERTS+ sl = raw[e * per:(e + 1) * per]+ # [out, in/8] packed u32: gate/up out=INTER in=DIM;+ # down out=DIM in=INTER.+ if proj == "down_proj":+ q = mx.array(sl.view("<u4")).reshape(DIM, INTER // 8)+ else:+ q = mx.array(sl.view("<u4")).reshape(INTER, DIM // 8)+ parts = {"weight": q}+ n_in = INTER if proj == "down_proj" else DIM+ for part in ("scales", "biases"):+ name = f"layers.0.mlp.switch_mlp.{proj}.{part}"+ raw = shard.raw(name)+ per = raw.size // N_EXPERTS+ sl = raw[e * per:(e + 1) * per]+ parts[part] = mx.array(sl.view("<u2")).view(+ mx.bfloat16).reshape(-1, n_in // 64)+ wcache[key] = mx.dequantize(+ parts["weight"], parts["scales"], parts["biases"],+ group_size=64, bits=4) # [out, in]+ return wcache[key]++ T, k = inds.shape+ rows = []+ for t in range(T):+ per = []+ for j in range(k):+ e = int(inds[t, j])+ gate = xf[t] @ w(e, "gate_proj").T+ up = xf[t] @ w(e, "up_proj").T+ z = nn.silu(gate) * up # swiglu+ per.append(z @ w(e, "down_proj").T)+ rows.append(mx.stack(per))+ return mx.stack(rows)+++def _inds(tokens=1, k=4):+ return mx.array(np.random.default_rng(0).integers(+ 0, N_EXPERTS, size=(tokens, k)), dtype=mx.int32)+++def test_exact_matches_reference(layer):+ lay, mm, fuse = layer+ x = mx.random.normal((1, 2, DIM)).astype(mx.float16)+ inds = _inds(tokens=2)+ out = lay(x, inds)+ assert out.shape == (1, 2, 4, DIM)+ ref = ref_moe(x[0], inds, mm, fuse)+ # gather_qmm evaluates with internal bf16 precision: a few elements+ # carry ~4.0 absolute error where |ref| is small, while the overall+ # error is only ≈0.24% relative L2. A per-element allclose would need+ # a tolerance loose enough to mask real bugs, so assert on relative+ # L2 instead: a gate/up swap or misaligned row layout (the bugs this+ # pins) shows up as ≈100% error, ~400x above the kernel noise floor.+ err = mx.sqrt(mx.sum((out[0].astype(mx.float32) - ref) ** 2)+ / mx.sum(ref ** 2))+ assert err < 1e-2, f"exact path diverges from dequant reference: {err}"+++def test_staged_matches_exact(layer):+ lay, _, _ = layer+ opts = _options(staged=True)+ # fresh layer, staged mode+ import tempfile, os+ with tempfile.TemporaryDirectory() as td:+ path = os.path.join(td, "w.safetensors")+ fp16 = _write_shard(path, lay._fuse_gu)+ mm = SafetensorsMmap(path)+ s = StreamingSwitchGLU([mm], 0, _spec(lay._fuse_gu), opts,+ shared_cache=SharedExpertCache(64))+ x = mx.random.normal((1, 1, DIM)).astype(mx.float16)+ inds = _inds(tokens=1)+ # exact reference from the same layer+ exact = lay(x, inds)+ # stage the same expert set synchronously, then consume via slots+ s.stage_experts(list(map(int, inds[0].tolist())))+ s.wait_staged()+ out = s(x, inds)+ assert out.shape == exact.shape+ assert mx.allclose(out, exact).item(), "staged != exact"+++def test_staged_replace_matches_exact(layer):+ lay, _, _ = layer+ import tempfile, os+ with tempfile.TemporaryDirectory() as td:+ path = os.path.join(td, "w.safetensors")+ _write_shard(path, lay._fuse_gu)+ mm = SafetensorsMmap(path)+ s = StreamingSwitchGLU(+ [mm], 0, _spec(lay._fuse_gu),+ _options(staged=True, staged_replace=True),+ shared_cache=SharedExpertCache(64))+ x = mx.random.normal((1, 1, DIM)).astype(mx.float16)+ staged = [0, 2, 5, 7]+ s.stage_experts(staged)+ s.wait_staged()+ out = s(x, None) # route through the whole set+ inds = mx.array([staged], dtype=mx.int32)+ exact = lay(x, inds)+ assert out.shape == exact.shape+ assert mx.allclose(out, exact).item(), "staged_replace != exact"+++def test_staged_fallback_when_empty(layer):+ lay, _, _ = layer+ import tempfile, os+ with tempfile.TemporaryDirectory() as td:+ path = os.path.join(td, "w.safetensors")+ _write_shard(path, lay._fuse_gu)+ mm = SafetensorsMmap(path)+ s = StreamingSwitchGLU([mm], 0, _spec(lay._fuse_gu),+ _options(staged=True),+ shared_cache=SharedExpertCache(64))+ x = mx.random.normal((1, 1, DIM)).astype(mx.float16)+ inds = _inds(tokens=1)+ # nothing staged -> falls back to the exact path+ out = s(x, inds)+ exact = lay(x, inds)+ assert mx.allclose(out, exact).item()+ st = s.stats()+ assert st["staged_fallback"] == 1+++def test_full_layer_prefill_matches_exact(layer):+ lay, _, _ = layer+ import tempfile, os+ with tempfile.TemporaryDirectory() as td:+ path = os.path.join(td, "w.safetensors")+ _write_shard(path, lay._fuse_gu)+ mm = SafetensorsMmap(path)+ s = StreamingSwitchGLU([mm], 0, _spec(lay._fuse_gu),+ _options(staged=True),+ shared_cache=SharedExpertCache(64))+ x = mx.random.normal((1, 4, DIM)).astype(mx.float16)+ # 3D router indices like the model forward ([B, T, k]): the FULL+ # branch keeps the [B, T, k, H] SwitchGLU contract without a batch+ # restore (matching the deployment's whole-layer path), so the+ # comparison must use the router's 3D shape, not the 2D _inds form.+ inds = mx.expand_dims(_inds(tokens=4), 0) # size 16 > 8 -> prefill path+ s.load_full_layer()+ out = s(x, inds)+ s.clear_full_layer()+ exact = lay(x, inds)+ assert out.shape == exact.shape+ assert mx.allclose(out, exact).item(), "full-layer != exact"+++def test_hot_stack_matches_exact(layer):+ lay, _, _ = layer+ import tempfile, os+ with tempfile.TemporaryDirectory() as td:+ path = os.path.join(td, "w.safetensors")+ _write_shard(path, lay._fuse_gu)+ mm = SafetensorsMmap(path)+ s = StreamingSwitchGLU([mm], 0, _spec(lay._fuse_gu),+ _options(staged=True, hot_per_layer=0),+ shared_cache=SharedExpertCache(64))+ x = mx.random.normal((1, 4, DIM)).astype(mx.float16)+ inds = _inds(tokens=4)+ s.load_hot_layer(n_hot=N_EXPERTS) # full coverage -> no misses+ s.materialize_hot()+ out = s(x, inds)+ exact = lay(x, inds)+ assert out.shape == exact.shape+ assert mx.allclose(out, exact).item(), "hot path != exact"+++def test_double_buffered_swap(layer):+ """stage_experts fills next; swap promotes; consume uses the new set."""+ lay, _, _ = layer+ import tempfile, os+ with tempfile.TemporaryDirectory() as td:+ path = os.path.join(td, "w.safetensors")+ _write_shard(path, lay._fuse_gu)+ mm = SafetensorsMmap(path)+ s = StreamingSwitchGLU(+ [mm], 0, _spec(lay._fuse_gu),+ _options(staged=True, staged_sync=False),+ shared_cache=SharedExpertCache(64))+ x = mx.random.normal((1, 1, DIM)).astype(mx.float16)+ first = [1, 3, 4, 6]+ second = [0, 2, 5, 7]+ s.stage_experts(first)+ s.swap_staged()+ out1 = s(x, mx.array([first], dtype=mx.int32))+ s.stage_experts(second)+ s.swap_staged()+ out2 = s(x, mx.array([second], dtype=mx.int32))+ ref1 = lay(x, mx.array([first], dtype=mx.int32))+ ref2 = lay(x, mx.array([second], dtype=mx.int32))+ assert mx.allclose(out1, ref1).item()+ assert mx.allclose(out2, ref2).item()