Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported
Python
#include "net8b.hpp"
#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <functional>
#include <limits>
#include <numeric>
#include <random>
#include <string>
#include <mlx/linalg.h>
#include "dense.hpp"
#include "qgemm.hpp"
namespace e0n {
namespace mx = mlx::core;
namespace {
using S = mx::Shape;
mx::array conv_from_torch(std::vector<uint8_t> &bytes, int64_t C, int64_t K) {
mx::array w(bytes.data(), mkshape({C, 1, K}), mx::bfloat16, [](void *) {});
return mx::swapaxes(w, 1, 2);
}
mx::array first_k(const mx::array &a, int64_t k) {
mx::Shape s1 = a.shape();
mx::Shape s0 = a.shape();
for (auto &v : s0) v = 0;
s1[s1.size() - 1] = static_cast<mx::Shape::value_type>(k);
return mx::slice(a, s0, s1);
}
// moe/routing.py group_select_from_logits(30-70) ≡ bailing_hybrid.py BailingGate.__call__
void select_group_gpu(const mx::array &logits, const mx::array *bias, int64_t k, int64_t n_group,
int64_t topk_group, double scaling, bool norm, mx::array &idx,
mx::array &w) {
auto scores = mx::sigmoid(mx::astype(logits, mx::float32)); // routing.py:41
mx::array select = scores; // 42
if (bias) select = scores + *bias;
const int64_t k_drop = n_group - topk_group; // 47
if (k_drop > 0) {
mx::Shape g = scores.shape(); // 49-50 (*b_shape, n_group, E/ng)
g[g.size() - 1] =
static_cast<mx::Shape::value_type>(g[g.size() - 1] / n_group);
g.insert(g.end() - 1, static_cast<mx::Shape::value_type>(n_group));
auto grouped = mx::reshape(select, g);
auto gs = mx::sum(mx::topk(grouped, 2, -1), std::vector<int>{-1}, false);
auto drop = first_k(mx::argpartition(gs, static_cast<int>(k_drop - 1), -1), k_drop); // 54-55
auto masked =
mx::put_along_axis(grouped, mx::expand_dims(drop, -1),
mx::array(-std::numeric_limits<float>::infinity(), mx::float32), -2);
select = mx::reshape(masked, scores.shape()); // 62
}
idx = first_k(mx::argpartition(-select, static_cast<int>(k - 1), -1), k); // 65
w = mx::take_along_axis(scores, idx, -1);
if (norm) w = w / (mx::sum(w, std::vector<int>{-1}, true) + mx::array(1e-20f, mx::float32)); // 67-68
w = w * mx::array(static_cast<float>(scaling), mx::float32); // 69
}
mx::array gate_logits32(const mx::array &m_in, const mx::array &gate_w) {
return mx::astype(mx::matmul(m_in, mx::transpose(gate_w)), mx::float32);
}
bool onehot_pos(const mx::array &idx, int64_t E, int64_t pos, mx::array &oh) {
if (idx.ndim() < 3) return false; // stager.py:45-46
const int64_t t = idx.shape(-2);
const int64_t lo = pos >= 0 ? pos : t + pos;
if (lo < 0 || lo >= t) return false; // 47-49
mx::Shape s0 = idx.shape(), s1 = idx.shape();
for (auto &v : s0) v = 0;
s0[s0.size() - 2] = static_cast<mx::Shape::value_type>(lo);
s1[s1.size() - 2] = static_cast<mx::Shape::value_type>(lo + 1);
auto row = mx::slice(idx, s0, s1); // 51 [..., 1, k]
auto eq = mx::equal(mx::expand_dims(row, -1), mx::arange(0, static_cast<int>(E))); // 52-53
oh = mx::astype(mx::sum(eq, std::vector<int>{-2}, false), mx::float32); // 53-54
return true;
}
mx::array rope_interleave_torch(const mx::array &x, int64_t Tpos_start, double theta,
int64_t D) {
// 98: freqs = 1.0 / (theta ** (arange(0,D,2,f32)/D))
auto ar = mx::divide(mx::arange(0, static_cast<int>(D), 2, mx::float32),
mx::array(static_cast<float>(D), mx::float32));
auto freqs = mx::divide(mx::array(1.0f, mx::float32),
mx::power(mx::array(static_cast<float>(theta), mx::float32), ar));
// 99: angles = outer(positions.f32, freqs);100: emb = cat(angles,angles)
auto positions =
mx::astype(mx::add(mx::arange(0, static_cast<int>(x.shape(2)), mx::int32),
mx::array(static_cast<int32_t>(Tpos_start), mx::int32)),
mx::float32);
auto angles = mx::outer(positions, freqs); // [L,D/2]
auto emb = mx::concatenate({angles, angles}, -1); // 100 [L,D]
auto cos = mx::cos(emb); // 101
auto sin = mx::sin(emb);
// 102-103: xi = x.reshape(B,H,L,D/2,2).transpose(0,1,2,4,3).reshape(B,H,L,D)
const int64_t B = x.shape(0), H = x.shape(1), Ld = x.shape(2);
auto xi = mx::reshape(
mx::transpose(mx::reshape(x, mkshape({B, H, Ld, D / 2, 2})), {0, 1, 2, 4, 3}),
mkshape({B, H, Ld, D}));
// 105: rh = cat(-xi[...,D/2:], xi[...,:D/2])
mx::Shape s0 = xi.shape(), s1 = xi.shape();
for (auto &v : s0) v = 0;
s1[3] = static_cast<mx::Shape::value_type>(D / 2);
auto lo = mx::slice(xi, s0, s1); // xi[..., :h]
s0[3] = static_cast<mx::Shape::value_type>(D / 2);
s1[3] = static_cast<mx::Shape::value_type>(D);
auto hi = mx::slice(xi, s0, s1); // xi[..., h:]
auto rh = mx::concatenate({mx::negative(hi), lo}, -1);
auto cosb = mx::reshape(cos, mkshape({1, 1, Ld, D}));
auto sinb = mx::reshape(sin, mkshape({1, 1, Ld, D}));
return mx::add(mx::multiply(xi, cosb), mx::multiply(rh, sinb));
}
std::vector<mx::array> moe_math_body(const std::vector<mx::array> &a) {
auto xe = mx::reshape(a[0], mkshape({a[0].shape(0), a[0].shape(1), 1, 1, a[0].shape(2)})); // layer.py:159
auto gq = [&](const mx::array &xx, int o) { // 160-167/169-171
return mx::gather_qmm(xx, a[o], a[o + 1], a[o + 2], std::nullopt, a[10], true, 64, 4,
"affine", false);
};
auto x_up = gq(xe, 1);
auto x_gate = gq(xe, 4);
auto yh = (x_gate * mx::sigmoid(x_gate)) * x_up; // _swiglu(x_up, x_gate)
auto z = gq(yh, 7);
return {mx::squeeze(z, -2)}; // 172
}
std::vector<mx::array> moe_math_sorted_body(const std::vector<mx::array> &a) {
auto gq = [&](const mx::array &xx, int o) { // 181-192
return mx::gather_qmm(xx, a[o], a[o + 1], a[o + 2], std::nullopt, a[10], true, 64, 4,
"affine", true);
};
auto x_up = gq(a[0], 1);
auto x_gate = gq(a[0], 4);
auto yh = (x_gate * mx::sigmoid(x_gate)) * x_up; // _swiglu
return {gq(yh, 7)};
}
ExpertPtrs bundle_ptrs(const Bundle &b) {
ExpertPtrs p;
p.uw = b.at(0).data<uint32_t>();
p.us = reinterpret_cast<const uint16_t *>(b.at(1).data<mx::bfloat16_t>());
p.ub = reinterpret_cast<const uint16_t *>(b.at(2).data<mx::bfloat16_t>());
p.gw = b.at(3).data<uint32_t>();
p.gs = reinterpret_cast<const uint16_t *>(b.at(4).data<mx::bfloat16_t>());
p.gb = reinterpret_cast<const uint16_t *>(b.at(5).data<mx::bfloat16_t>());
p.dw = b.at(6).data<uint32_t>();
p.ds = reinterpret_cast<const uint16_t *>(b.at(7).data<mx::bfloat16_t>());
p.db = reinterpret_cast<const uint16_t *>(b.at(8).data<mx::bfloat16_t>());
return p;
}
} // namespace
std::unique_ptr<Net8b> Net8b::create(std::shared_ptr<const ModelFiles> files,
std::shared_ptr<const ExpertMap> emap, std::string &err,
std::shared_ptr<const lora::Bundle> lo) {
std::unique_ptr<Net8b> n(new Net8b());
n->lo_ = lo;
n->files_ = files;
n->emap_ = emap;
if (!tier_routing("edge0-8b", n->reg_)) { err = "registry missing edge0-8b"; return nullptr; }
int64_t cfg_layers = 0;
if (files->config_int("num_hidden_layers", cfg_layers) && cfg_layers != n->reg_.num_layers) {
err = "config layer count does not match registry";
return nullptr;
}
auto ci = [&](const char *k, int64_t def, int64_t &o) {
int64_t v;
o = files->config_int(k, v) ? v : def;
};
auto cd = [&](const char *k, double def, double &o) {
double v;
o = files->config_double(k, v) ? v : def;
};
ci("hidden_size", 1536, n->H_);
ci("vocab_size", 157184, n->V_);
ci("num_attention_heads", 16, n->nh_);
ci("head_dim", 128, n->hd_);
ci("short_conv_kernel_size", 4, n->conv_k_);
ci("moe_intermediate_size", 512, n->moe_ffn_);
{
int64_t shi = 512, shn = 1;
ci("moe_shared_expert_intermediate_size", 512, shi);
ci("num_shared_experts", 1, shn);
n->sh_ffn_ = shi * shn;
}
ci("intermediate_size", 4608, n->dense_ffn_);
ci("q_lora_rank", 256, n->qlora_);
ci("kv_lora_rank", 512, n->kvlora_);
ci("qk_nope_head_dim", 128, n->nope_);
ci("qk_rope_head_dim", 64, n->roped_);
ci("v_head_dim", 128, n->vhd_);
cd("rms_norm_eps", 1e-6, n->eps_);
cd("rope_theta", 6000000.0, n->rope_theta_);
cd("kda_lower_bound", -5.0, n->kda_lb_);
n->proj_ = n->nh_ * n->hd_;
int64_t qbits = 4, qgroup = 64;
if (files->config_quant(qbits, qgroup) && qgroup > 0) n->group_ = qgroup;
int64_t interval = 4, dense_prefix_cfg = 1;
ci("layer_group_size", 4, interval);
ci("first_k_dense_replace", 1, dense_prefix_cfg);
if (dense_prefix_cfg != n->reg_.dense_prefix) { err = "config dense prefix does not match registry"; return nullptr; }
const int64_t NL = n->reg_.num_layers;
{
static const std::string kKdaSource = R"gdn(
auto n = thread_position_in_grid.z;
auto b_idx = n / Hv;
auto hv_idx = n % Hv;
auto hk_idx = hv_idx / (Hv / Hk);
constexpr int n_per_t = Dk / 32;
// q, k: [B, T, Hk, Dk]
auto q_ = q + b_idx * T * Hk * Dk + hk_idx * Dk;
auto k_ = k + b_idx * T * Hk * Dk + hk_idx * Dk;
// v, y: [B, T, Hv, Dv]
auto v_ = v + b_idx * T * Hv * Dv + hv_idx * Dv;
y += b_idx * T * Hv * Dv + hv_idx * Dv;
auto dk_idx = thread_position_in_threadgroup.x;
auto dv_idx = thread_position_in_grid.y;
// state_in, state_out: [B, Hv, Dv, Dk]
auto i_state = state_in + (n * Dv + dv_idx) * Dk;
auto o_state = state_out + (n * Dv + dv_idx) * Dk;
float state[n_per_t];
for (int i = 0; i < n_per_t; ++i) {
auto s_idx = n_per_t * dk_idx + i;
state[i] = static_cast<float>(i_state[s_idx]);
}
// g: [B, T, Hv, Dk]
auto g_ = g + (b_idx * T * Hv + hv_idx) * Dk;
auto beta_ = beta + b_idx * T * Hv;
for (int t = 0; t < T; ++t) {
if (true) {
float kv_mem = 0.0f;
for (int i = 0; i < n_per_t; ++i) {
auto s_idx = n_per_t * dk_idx + i;
state[i] = state[i] * g_[s_idx];
kv_mem += state[i] * k_[s_idx];
}
kv_mem = simd_sum(kv_mem);
auto delta = (v_[dv_idx] - kv_mem) * beta_[hv_idx];
float out = 0.0f;
for (int i = 0; i < n_per_t; ++i) {
auto s_idx = n_per_t * dk_idx + i;
state[i] = state[i] + k_[s_idx] * delta;
out += state[i] * q_[s_idx];
}
out = simd_sum(out);
if (thread_index_in_simdgroup == 0) {
y[dv_idx] = static_cast<InT>(out);
}
}
// Increment data pointers to next time step
q_ += Hk * Dk;
k_ += Hk * Dk;
v_ += Hv * Dv;
y += Hv * Dv;
g_ += Hv * Dk;
beta_ += Hv;
}
for (int i = 0; i < n_per_t; ++i) {
auto s_idx = n_per_t * dk_idx + i;
o_state[s_idx] = static_cast<InT>(state[i]);
}
)gdn";
n->kda_kernel_ = mx::fast::metal_kernel(
"gated_delta_step_vec", {"q", "k", "v", "g", "beta", "state_in", "T"},
{"y", "state_out"}, kKdaSource);
}
n->layers_.resize(static_cast<size_t>(NL));
const std::string M = "model.layers.";
for (int64_t li = 0; li < NL; li++) {
Layer &L = n->layers_[static_cast<size_t>(li)];
const std::string P = M + std::to_string(li);
L.is_mla = ((li + 1) % interval) == 0 || li >= NL / interval * interval;
L.has_moe = li >= n->reg_.dense_prefix;
auto q = [&](const std::string &name, QuantLin &dst) {
if (!dst.load(*files, P + name, err)) { err += " @" + std::to_string(li); return false; }
n->dense_bytes_ += dst.wb.size() + dst.sb.size() + dst.bb.size();
return true;
};
auto vec = [&](const std::string &name, std::vector<float> &dstv) {
DenseF32 d;
if (!read_dense_f32(*files, P + name, d, err)) { err += " @" + std::to_string(li); return false; }
dstv = std::move(d.v);
n->dense_bytes_ += dstv.size() * 4;
return true;
};
if (!vec(".input_layernorm.weight", L.in_ln_w) ||
!vec(".post_attention_layernorm.weight", L.post_ln_w))
return nullptr;
if (!L.is_mla) {
if (!q(".attention.q_proj.weight", L.k_q) || !q(".attention.k_proj.weight", L.k_k) ||
!q(".attention.v_proj.weight", L.k_v) || !q(".attention.f_proj.weight", L.k_f) ||
!q(".attention.g_proj.weight", L.k_g) || !q(".attention.b_proj.weight", L.k_b) ||
!q(".attention.o_proj.weight", L.k_o) ||
!vec(".attention.A_log", L.A_log) || !vec(".attention.dt_bias", L.dt_bias) ||
!vec(".attention.o_norm.weight", L.onorm_w))
return nullptr;
if (static_cast<int64_t>(L.A_log.size()) != n->nh_ ||
static_cast<int64_t>(L.dt_bias.size()) != n->proj_ ||
static_cast<int64_t>(L.onorm_w.size()) != n->hd_) {
err = "KDA parameter vector length mismatch";
return nullptr;
}
if (!files->read_tensor(P + ".attention.q_conv1d.weight", L.conv_qb, err) ||
!files->read_tensor(P + ".attention.k_conv1d.weight", L.conv_kb, err) ||
!files->read_tensor(P + ".attention.v_conv1d.weight", L.conv_vb, err))
return nullptr;
auto ct = files->find(P + ".attention.q_conv1d.weight")->shape;
if (ct.size() != 3 || ct[0] != n->proj_ || ct[2] != n->conv_k_) {
err = "KDA conv shape is not [proj,1,ksize]";
return nullptr;
}
L.conv_q = conv_from_torch(L.conv_qb, n->proj_, n->conv_k_);
L.conv_k = conv_from_torch(L.conv_kb, n->proj_, n->conv_k_);
L.conv_v = conv_from_torch(L.conv_vb, n->proj_, n->conv_k_);
n->dense_bytes_ += L.conv_qb.size() + L.conv_kb.size() + L.conv_vb.size();
} else {
if (!q(".attention.q_a_proj.weight", L.m_qa) || !q(".attention.q_b_proj.weight", L.m_qb) ||
!q(".attention.kv_a_proj_with_mqa.weight", L.m_kva) ||
!q(".attention.kv_b_proj.weight", L.m_kvb) || !q(".attention.g_proj.weight", L.m_g) ||
!q(".attention.dense.weight", L.m_dense) ||
!vec(".attention.q_a_layernorm.weight", L.m_qaln) ||
!vec(".attention.kv_a_layernorm.weight", L.m_kvaln))
return nullptr;
if (static_cast<int64_t>(L.m_qaln.size()) != n->qlora_ ||
static_cast<int64_t>(L.m_kvaln.size()) != n->kvlora_) {
err = "MLA LoRA normalize length mismatch";
return nullptr;
}
}
if (!L.has_moe) {
if (!q(".mlp.gate_proj.weight", L.d_gate) || !q(".mlp.up_proj.weight", L.d_up) ||
!q(".mlp.down_proj.weight", L.d_down))
return nullptr;
continue;
}
DenseF32 g;
const std::string gw = P + ".mlp.gate.weight";
const TensorInfo *gi = files->find(gw);
if (!gi) { err = "missing gate"; return nullptr; }
if (gi->dtype == "U32") {
int64_t bits2 = 4, gs2 = 64;
if (!files->config_quant_for(gw, bits2, gs2) ||
!read_dense_dequant(*files, gw, bits2, gs2, g, err))
return nullptr;
} else if (!read_dense_f32(*files, gw, g, err)) {
return nullptr;
}
if (g.rows != n->reg_.num_experts || g.cols != n->H_) { err = "gate shape mismatch"; return nullptr; }
L.gate_h = std::move(g.v);
L.gate_bf16 = gi->dtype != "U32";
n->dense_bytes_ += L.gate_h.size() * 4;
{
DenseF32 eb;
if (!read_dense_f32(*files, P + ".mlp.gate.expert_bias", eb, err)) return nullptr;
if (static_cast<int64_t>(eb.v.size()) != n->reg_.num_experts) {
err = "expert_bias length mismatch";
return nullptr;
}
L.gate_bias.assign(eb.v.begin(), eb.v.end());
n->dense_bytes_ += L.gate_bias.size() * 4;
}
if (!q(".mlp.shared_experts.gate_proj.weight", L.sh_gate) ||
!q(".mlp.shared_experts.up_proj.weight", L.sh_up) ||
!q(".mlp.shared_experts.down_proj.weight", L.sh_down))
return nullptr;
}
if (!n->embed_.load(*files, "model.word_embeddings.weight", err)) return nullptr;
if (!n->lm_head_.load(*files, "lm_head.weight", err)) return nullptr;
n->dense_bytes_ += n->embed_.wb.size() + n->embed_.sb.size() + n->embed_.bb.size() +
n->lm_head_.wb.size() + n->lm_head_.sb.size() + n->lm_head_.bb.size();
{
DenseF32 f;
if (!read_dense_f32(*files, "model.norm.weight", f, err)) return nullptr;
n->final_norm_w_ = std::move(f.v);
}
n->pr_ = std::make_unique<Prerouter>();
if (!Prerouter::load(*files, n->reg_, n->H_, *n->pr_, err)) return nullptr;
{
const int64_t NL2 = n->reg_.num_layers;
n->io_ = std::make_unique<ExpertIo>(files.get(), 64, 48, 8, 4);
for (int64_t li = 0; li < NL2; li++) {
if (!n->layers_[size_t(li)].has_moe) continue;
ExpertIo::LayerCfg c;
c.layer = int(li);
c.prefix = emap->layer_prefix(int(li));
c.E = n->reg_.num_experts;
c.H = n->H_;
c.F = n->moe_ffn_;
c.group = n->group_;
c.staged_n = int(n->reg_.top_k);
c.prefill_hot = (int)n->reg_.prefill_hot;
c.full_layer_prefill = n->reg_.full_layer_prefill;
n->io_->add_layer(c);
}
}
if (exec_mode() == ExecMode::Batched) {
const char *cl = getenv("MLX_CACHE_LIMIT_MB");
mx::set_cache_limit(static_cast<size_t>(cl ? atoi(cl) : 256) << 20);
const int64_t E = n->reg_.num_experts, NL = n->reg_.num_layers;
n->m_in_cache_.assign(NL, mx::array(0.f));
n->last_topk_.assign(NL, mx::array(0.f));
n->prev_topk_oh_.assign(NL, mx::array(0.f));
n->pg_logits_.assign(NL, mx::array(0.f));
n->gate_w32_.assign(NL, mx::array(0.f));
n->gate_b32_.assign(NL, mx::array(0.f));
for (int64_t li = 0; li < NL; li++) {
Layer &L = n->layers_[static_cast<size_t>(li)];
if (!L.has_moe) continue;
auto gw =
mx::array(L.gate_h.data(), mkshape({E, n->H_}), mx::float32, [](void *) {});
n->gate_w32_[li] = L.gate_bf16 ? mx::astype(gw, mx::bfloat16) : gw;
std::vector<float> b32(static_cast<size_t>(E));
for (int64_t e = 0; e < E; e++)
b32[static_cast<size_t>(e)] = static_cast<float>(L.gate_bias[static_cast<size_t>(e)]);
n->gate_b32_[li] = mx::array(b32.begin(), mkshape({E}), mx::float32);
}
const int64_t nH = n->pr_->n_heads(), f_in = n->pr_->stacked_f_in(),
prh = n->pr_->stacked_prh(), hE = n->pr_->stacked_E();
if (nH > 0) {
auto wrap16 = [&](const std::vector<float> &v, std::initializer_list<int64_t> dims) {
return mx::astype(mx::array((void *)v.data(), mkshape(dims), mx::float32, [](void *) {}),
mx::float16);
};
n->hw1_ = wrap16(n->pr_->stacked_fc1(), {nH, f_in, prh});
n->hw2_ = wrap16(n->pr_->stacked_fc2(), {nH, prh, hE});
n->hwl_ = wrap16(n->pr_->stacked_lin(), {nH, f_in, hE});
mx::eval(n->hw1_, n->hw2_, n->hwl_);
}
try {
auto fn = mx::compile(&moe_math_body);
const int64_t F = n->moe_ffn_, gsh = n->H_ / 64, gsf = F / 64;
std::vector<mx::array> d = {
mx::zeros(mkshape({1, 1, n->H_}), mx::bfloat16),
mx::zeros(mkshape({2, F, n->H_ / 8}), mx::uint32),
mx::zeros(mkshape({2, F, gsh}), mx::bfloat16),
mx::zeros(mkshape({2, F, gsh}), mx::bfloat16),
mx::zeros(mkshape({2, F, n->H_ / 8}), mx::uint32),
mx::zeros(mkshape({2, F, gsh}), mx::bfloat16),
mx::zeros(mkshape({2, F, gsh}), mx::bfloat16),
mx::zeros(mkshape({2, n->H_, F / 8}), mx::uint32),
mx::zeros(mkshape({2, n->H_, gsf}), mx::bfloat16),
mx::zeros(mkshape({2, n->H_, gsf}), mx::bfloat16),
mx::zeros(mkshape({1, 1, 2}), mx::int32)};
auto out = fn(d);
mx::eval(out[0]);
n->moe_math_ = std::move(fn);
} catch (const std::exception &ex) {
std::fprintf(stderr, "edge0-8b: mlx compile unavailable (%s), MoE math falls back to eager\n",
ex.what());
n->moe_math_ = &moe_math_body;
}
try {
auto fn = mx::compile(&moe_math_sorted_body);
const int64_t F = n->moe_ffn_, gsh = n->H_ / 64, gsf = F / 64;
std::vector<mx::array> d = {
mx::zeros(mkshape({16, 1, n->H_}), mx::bfloat16),
mx::zeros(mkshape({2, F, n->H_ / 8}), mx::uint32),
mx::zeros(mkshape({2, F, gsh}), mx::bfloat16),
mx::zeros(mkshape({2, F, gsh}), mx::bfloat16),
mx::zeros(mkshape({2, F, n->H_ / 8}), mx::uint32),
mx::zeros(mkshape({2, F, gsh}), mx::bfloat16),
mx::zeros(mkshape({2, F, gsh}), mx::bfloat16),
mx::zeros(mkshape({2, n->H_, F / 8}), mx::uint32),
mx::zeros(mkshape({2, n->H_, gsf}), mx::bfloat16),
mx::zeros(mkshape({2, n->H_, gsf}), mx::bfloat16),
mx::zeros(mkshape({16}), mx::int32)};
auto out = fn(d);
mx::eval(out[0]);
n->moe_math_sorted_ = std::move(fn);
} catch (const std::exception &ex) {
std::fprintf(stderr, "edge0-8b: mlx compile unavailable (%s), sorted MoE math falls back to eager\n",
ex.what());
n->moe_math_sorted_ = &moe_math_sorted_body;
}
}
{
const char *pw = getenv("EDGE0_PREWARM");
if (!pw) pw = getenv("LING_PREWARM");
if (pw && !strcmp(pw, "1")) {
auto tp0 = std::chrono::steady_clock::now();
try {
for (const auto &sn : files->shard_names()) { // shard.advise_willneed + seq_read
n->io_->shard_advise_whole(sn);
n->io_->shard_seq_read(sn);
}
std::string perr;
std::vector<float> lg;
std::vector<RouteRec> recs;
State8b pst;
n->init_state(pst);
const std::vector<int32_t> dummy(4, 0); // prefill(dummy*4)
n->prefill(dummy, pst, lg, recs, perr);
n->step(0, pst, lg, recs, perr); // step(dummy[0])
n->reset_state(pst);
std::fprintf(stderr, "[edge0-8b] prewarm done in %.1fs\n",
std::chrono::duration<double>(std::chrono::steady_clock::now() - tp0).count());
} catch (...) { /* advisory */ }
}
}
if (n->lo_) {
const int nl = n->reg_.num_layers, mla = nl / 4;
const int want = 7 * (nl - mla) + 4 * mla + 3;
if (n->lo_->n_bound() != want) {
err = "LoRA bind count " + std::to_string(n->lo_->n_bound()) + " != structural count " +
std::to_string(want);
return nullptr;
}
n->dense_bytes_ += static_cast<uint64_t>(n->lo_->bytes());
}
return n;
}
void Net8b::init_state(State8b &st) const {
st.pr.init(reg_);
const size_t NL = layers_.size();
st.cq.assign(NL, mx::array(0.f));
st.ck.assign(NL, mx::array(0.f));
st.cv.assign(NL, mx::array(0.f));
st.ssm.assign(NL, mx::array(0.f));
st.kc.assign(NL, mx::array(0.f));
st.vc.assign(NL, mx::array(0.f));
st.is_mla.assign(NL, 0);
for (size_t i = 0; i < NL; i++) st.is_mla[i] = layers_[i].is_mla ? 1 : 0;
st.position = 0;
st.started = true;
reset_graph_caches();
}
void Net8b::reset_state(State8b &st) const {
st.pr.reset();
const size_t NL = layers_.size();
st.cq.assign(NL, mx::array(0.f));
st.ck.assign(NL, mx::array(0.f));
st.cv.assign(NL, mx::array(0.f));
st.ssm.assign(NL, mx::array(0.f));
st.kc.assign(NL, mx::array(0.f));
st.vc.assign(NL, mx::array(0.f));
st.position = 0;
reset_graph_caches();
}
bool Net8b::layer_fwd(Layer &L, int li, mx::array &x, State8b &st, bool decode, bool prefill_seed,
RouteRec &rec, std::string &err) {
const int64_t T = x.shape(1);
const auto dt = x.dtype();
auto norm_w = [&](const std::vector<float> &v) {
return mx::astype(wrap_f32(v, mkshape({static_cast<int64_t>(v.size())})), dt);
};
mx::array xn = e0_rms_norm(x, norm_w(L.in_ln_w), eps_);
mx::array r{0.f};
if (!L.is_mla) {
const auto *lo = lo_.get();
mx::array q_lin = e0n::lora::apply_lora(lo, L.k_q, xn, li, e0n::lora::S8B_Q),
k_lin = e0n::lora::apply_lora(lo, L.k_k, xn, li, e0n::lora::S8B_K),
v_lin = e0n::lora::apply_lora(lo, L.k_v, xn, li, e0n::lora::S8B_V);
auto conv_step = [&](const mx::array &lin, mx::array &state, const mx::array &w) {
if (state.ndim() == 0)
state = mx::zeros(mkshape({1, conv_k_ - 1, proj_}), dt);
mx::array cat = mx::concatenate({state, lin}, 1); // [1, K-1+T, proj]
state = mx::slice(cat, mkshape({0, T, 0}), mkshape({1, T + conv_k_ - 1, proj_}));
return e0_silu(mx::conv1d(cat, w, 1, 0, 1, static_cast<int>(proj_))); // [1,T,proj]
};
mx::array qc = conv_step(q_lin, st.cq[li], L.conv_q);
mx::array kc = conv_step(k_lin, st.ck[li], L.conv_k);
mx::array vc = conv_step(v_lin, st.cv[li], L.conv_v);
mx::array qf = mx::astype(mx::reshape(qc, mkshape({1, T, nh_, hd_})), mx::float32);
mx::array kf = mx::astype(mx::reshape(kc, mkshape({1, T, nh_, hd_})), mx::float32);
mx::array vf = mx::astype(mx::reshape(vc, mkshape({1, T, nh_, hd_})), mx::float32);
float sc = 1.0f / std::sqrt(static_cast<float>(hd_));
qf = sc * qf / (mx::linalg::norm(qf, 2.0, -1, true) + 1e-6f);
kf = kf / (mx::linalg::norm(kf, 2.0, -1, true) + 1e-6f);
mx::array gate = mx::reshape(
e0n::lora::apply_lora(lo, L.k_g, xn, li, e0n::lora::S8B_G), mkshape({1, T, nh_, hd_}));
mx::array f = mx::astype(mx::reshape(e0n::lora::apply_lora(lo, L.k_f, xn, li, e0n::lora::S8B_F),
mkshape({1, T, nh_, hd_})),
mx::float32);
mx::array dtb = mx::reshape(wrap_f32(L.dt_bias, mkshape({nh_, hd_})), mkshape({1, 1, nh_, hd_}));
mx::array alog = wrap_f32(L.A_log, mkshape({nh_, 1}));
mx::array glog = static_cast<float>(kda_lb_) * mx::sigmoid(mx::exp(alog) * (f + dtb));
mx::array gdec = mx::exp(glog);
mx::array beta = mx::sigmoid(mx::astype(
e0n::lora::apply_lora(lo, L.k_b, xn, li, e0n::lora::S8B_B), mx::float32)); // [1,T,H]
if (st.ssm[li].ndim() == 0)
st.ssm[li] = mx::zeros(mkshape({1, nh_, hd_, hd_}), mx::float32);
auto kouts = kda_kernel_(
{qf, kf, vf, gdec, beta, st.ssm[li], mx::array((int)T, mx::int32)},
{mkshape({1, T, nh_, hd_}), st.ssm[li].shape()},
{mx::float32, mx::float32},
{32, static_cast<int>(hd_), static_cast<int>(nh_)}, // grid=(32, Dv, B*Hv),B=1
{32, 4, 1}, // threadgroup(gated_delta.py:206)
{{"InT", mx::float32},
{"Dk", static_cast<int>(hd_)},
{"Dv", static_cast<int>(hd_)},
{"Hk", static_cast<int>(nh_)},
{"Hv", static_cast<int>(nh_)}},
std::nullopt, false, {});
mx::array y = kouts[0]; // [1,T,nh,hd] f32
st.ssm[li] = kouts[1];
mx::array gated =
e0_rms_norm(mx::astype(y, dt), norm_w(L.onorm_w), eps_) * mx::sigmoid(gate);
r = e0n::lora::apply_lora(lo, L.k_o, mx::reshape(gated, mkshape({1, T, proj_})), li,
e0n::lora::S8B_O);
} else {
mx::array q_lat = e0_rms_norm(L.m_qa.apply(xn), norm_w(L.m_qaln), eps_);
mx::array qfull = mx::transpose(
mx::reshape(e0n::lora::apply_lora(lo_.get(), L.m_qb, q_lat, li, e0n::lora::S8B_M_QB),
mkshape({1, T, nh_, nope_ + roped_})),
{0, 2, 1, 3});
mx::eval(qfull);
auto qs = mx::split(qfull, S{static_cast<int32_t>(nope_)}, -1);
mx::array q_nope = qs[0], q_pe = qs[1];
mx::array comp = L.m_kva.apply(xn); // [1,T,kvlora+rope]
mx::eval(comp);
auto cs = mx::split(comp, S{static_cast<int32_t>(kvlora_)}, -1);
mx::array kv_lat = e0_rms_norm(cs[0], norm_w(L.m_kvaln), eps_);
mx::array kv = mx::transpose(
mx::reshape(
e0n::lora::apply_lora(lo_.get(), L.m_kvb, kv_lat, li, e0n::lora::S8B_M_KVB),
mkshape({1, T, nh_, nope_ + vhd_})),
{0, 2, 1, 3});
auto ks = mx::split(kv, S{static_cast<int32_t>(nope_)}, -1);
mx::array k_nope = ks[0], values = ks[1];
mx::array k_pe = mx::broadcast_to(
rope_interleave_torch(
mx::reshape(mx::slice(comp, mkshape({0, 0, kvlora_}),
mkshape({1, T, kvlora_ + roped_})),
mkshape({1, 1, T, roped_})),
st.position, rope_theta_, roped_),
mkshape({1, nh_, T, roped_}));
q_pe = rope_interleave_torch(q_pe, st.position, rope_theta_, roped_);
mx::array queries = mx::concatenate({q_nope, q_pe}, -1); // [1,nh,T,192]
mx::array keys = mx::concatenate({k_nope, k_pe}, -1);
if (st.kc[li].ndim() == 0) {
st.kc[li] = keys;
st.vc[li] = values;
} else {
st.kc[li] = mx::concatenate({st.kc[li], keys}, 2);
st.vc[li] = mx::concatenate({st.vc[li], values}, 2);
}
float scale = 1.0f / std::sqrt(static_cast<float>(nope_ + roped_));
mx::array o = mx::fast::scaled_dot_product_attention(
queries, st.kc[li], st.vc[li], scale, "causal"); // [1,nh,T,vhd]
o = mx::reshape(mx::transpose(o, {0, 2, 1, 3}), mkshape({1, T, nh_ * vhd_}));
mx::array gate = mx::sigmoid(
e0n::lora::apply_lora(lo_.get(), L.m_g, xn, li, e0n::lora::S8B_M_G)); // [1,T,nh] head-wise
o = mx::reshape(mx::reshape(o, mkshape({1, T, nh_, vhd_})) *
mx::expand_dims(gate, -1),
mkshape({1, T, nh_ * vhd_}));
r = e0n::lora::apply_lora(lo_.get(), L.m_dense, o, li, e0n::lora::S8B_M_DENSE);
}
mx::array h = x + r;
mx::array m_in = e0_rms_norm(h, norm_w(L.post_ln_w), eps_);
if (!L.has_moe) {
mx::array y =
e0n::lora::apply_lora(
lo_.get(), L.d_down,
e0_silu(e0n::lora::apply_lora(lo_.get(), L.d_gate, m_in, li, e0n::lora::S8B_D_GATE)) *
e0n::lora::apply_lora(lo_.get(), L.d_up, m_in, li, e0n::lora::S8B_D_UP),
li, e0n::lora::S8B_D_DOWN);
x = h + y;
if (clip_ > 0)
x = mx::where(mx::isnan(x), mx::zeros_like(x),
mx::clip(x, mx::array(static_cast<float>(-clip_)),
mx::array(static_cast<float>(clip_))));
if (!(exec_mode() == ExecMode::Batched && decode && T == 1))
mx::eval(x);
return true;
}
// bailing_hybrid.py BailingSparseMoE.__call__(679-695)/DecoderLayer.__call__(739-762):
if (exec_mode() == ExecMode::Batched && decode && T == 1) {
const int64_t k = reg_.top_k;
io_->note_call(li);
const bool owner = li >= reg_.first_owner && li <= reg_.last_owner;
const bool consumer = li >= reg_.first_head_consumer && li <= reg_.last_head_consumer;
if (owner) m_in_cache_[static_cast<size_t>(li)] = m_in;
io_->wait_staged(li);
const auto &st_st = io_->staged(li);
const bool use_pg =
consumer && pg_logits_[static_cast<size_t>(li)].ndim() > 0 &&
!st.pr.pred[static_cast<size_t>(li)].inds.empty() &&
st_st.active && st_st.exp == st.pr.pred[static_cast<size_t>(li)].inds;
mx::array idx(0.f), w(0.f), routed(0.f), teach(0.f);
rec.layer = li;
if (use_pg) {
select_group_gpu(pg_logits_[static_cast<size_t>(li)], nullptr, k, reg_.n_group,
reg_.topk_group, reg_.routed_scaling, reg_.norm_topk_prob, idx, w);
if (owner) {
auto gl = gate_logits32(m_in, gate_w32_[static_cast<size_t>(li)]);
mx::array w_unused(0.f);
select_group_gpu(gl, &gate_b32_[static_cast<size_t>(li)], k, reg_.n_group,
reg_.topk_group, reg_.routed_scaling, reg_.norm_topk_prob, teach,
w_unused);
}
rec.predicted = true;
rec.sel = st.pr.pred[static_cast<size_t>(li)];
st.pr.pred_avail[static_cast<size_t>(li)] = false;
auto asm_ = io_->asm_for(li, st_st);
auto local = mx::take(asm_.slot_of, idx);
std::vector<mx::array> args{m_in};
for (auto &a : asm_.wargs) args.push_back(a);
args.push_back(local);
auto &cx = exec_counters();
cx.qmm_calls.fetch_add(3, std::memory_order_relaxed);
cx.qmm_experts.fetch_add(3 * static_cast<uint64_t>(asm_.exp.size()), std::memory_order_relaxed);
cx.qmm_weight_bytes.fetch_add(
static_cast<uint64_t>(asm_.exp.size()) *
(2 * (moe_ffn_ * (H_ / 8) * 4 + 2 * moe_ffn_ * (H_ / group_) * 2) +
(H_ * (moe_ffn_ / 8) * 4 + 2 * H_ * (moe_ffn_ / group_) * 2)),
std::memory_order_relaxed);
routed = moe_math_(args)[0]; // layer.py:1359 self._moe_math(x, *wargs, local2d)
} else {
const int F = static_cast<int>(moe_ffn_);
const int gsh = static_cast<int>(H_ / group_), gsf = static_cast<int>(moe_ffn_ / group_);
auto gl = gate_logits32(m_in, gate_w32_[static_cast<size_t>(li)]);
select_group_gpu(gl, &gate_b32_[static_cast<size_t>(li)], k, reg_.n_group, reg_.topk_group,
reg_.routed_scaling, reg_.norm_topk_prob, idx, w);
if (owner) teach = idx;
mx::eval(idx, w);
exec_counters().host_syncs.fetch_add(1, std::memory_order_relaxed);
auto idxh = mx::copy(mx::astype(idx, mx::int32), mx::default_stream(mx::Device::cpu));
auto wh = mx::copy(mx::astype(w, mx::float32), mx::default_stream(mx::Device::cpu));
mx::eval(idxh, wh);
const int32_t *ip = idxh.data<int32_t>();
const float *wp = wh.data<float>();
std::vector<int64_t> flat(ip, ip + k);
std::vector<int64_t> uniq = flat;
std::sort(uniq.begin(), uniq.end());
uniq.erase(std::unique(uniq.begin(), uniq.end()), uniq.end());
std::vector<int64_t> local_h(flat.size());
for (size_t j = 0; j < flat.size(); j++)
local_h[j] = static_cast<int32_t>(
std::lower_bound(uniq.begin(), uniq.end(), flat[j]) - uniq.begin());
auto bundles = io_->get_bundles(li, uniq);
if (bundles.size() != uniq.size()) {
err = "expert load missing items L" + std::to_string(li);
return false;
}
exec_counters().leases.fetch_add(uniq.size(), std::memory_order_relaxed);
for (auto &[e, b] : bundles) step_bundles_.push_back(b);
std::vector<mx::array> args{m_in};
for (int p = 0; p < 9; p++) {
std::vector<mx::array> rows;
for (auto e : uniq) rows.push_back(bundles[e]->at(p));
args.push_back(mx::stack(rows)); // [U,...](1380)
}
args.push_back(mx::array(local_h.begin(), mkshape({1, 1, k}), mx::int32)); // 1371 core.array(list)
auto &cx = exec_counters();
cx.qmm_calls.fetch_add(3, std::memory_order_relaxed);
cx.qmm_experts.fetch_add(3 * static_cast<uint64_t>(uniq.size()), std::memory_order_relaxed);
cx.qmm_weight_bytes.fetch_add(
static_cast<uint64_t>(uniq.size()) *
(2 * (F * (H_ / 8) * 4 + 2 * F * gsh * 2) + (H_ * (F / 8) * 4 + 2 * H_ * gsf * 2)),
std::memory_order_relaxed);
routed = moe_math_(args)[0];
rec.predicted = false;
rec.sel.inds = uniq;
rec.sel.scores.clear();
for (int64_t e : uniq) {
double s = 0.0;
for (int64_t j = 0; j < k; j++)
if (flat[j] == e) s = wp[j];
rec.sel.scores.push_back(s);
}
}
if (owner) last_topk_[static_cast<size_t>(li)] = use_pg ? teach : idx;
auto out = mx::sum(routed * mx::astype(mx::expand_dims(w, -1), routed.dtype()),
std::vector<int>{-2}, false);
mx::array sh =
L.sh_down.apply(e0_silu(L.sh_gate.apply(m_in)) * L.sh_up.apply(m_in)); // BailingMLP:503
x = h + out + sh;
if (clip_ > 0)
x = mx::where(mx::isnan(x), mx::zeros_like(x),
mx::clip(x, mx::array(static_cast<float>(-clip_)),
mx::array(static_cast<float>(clip_))));
return true;
}
// before_layer_cb load-drop hooks.py:35-58 window=1 + async_eval_per_layer
if (exec_mode() == ExecMode::Batched) {
const int64_t k = reg_.top_k;
const int64_t E = reg_.num_experts;
io_->note_call(li);
const bool owner = li >= reg_.first_owner && li <= reg_.last_owner;
if (owner) m_in_cache_[static_cast<size_t>(li)] = m_in;
e0n::TimeSink gt_cpu(e0n::timing_counters().gate_cpu_ns);
auto gl = gate_logits32(m_in, gate_w32_[static_cast<size_t>(li)]); // :527
mx::array idx(0.f), w(0.f);
select_group_gpu(gl, &gate_b32_[static_cast<size_t>(li)], k, reg_.n_group, reg_.topk_group,
reg_.routed_scaling, reg_.norm_topk_prob, idx, w);
if (owner) last_topk_[static_cast<size_t>(li)] = idx;
mx::eval(idx, w);
exec_counters().host_syncs.fetch_add(1, std::memory_order_relaxed);
auto idxh = mx::copy(mx::astype(idx, mx::int32), mx::default_stream(mx::Device::cpu));
auto wh = mx::copy(mx::astype(w, mx::float32), mx::default_stream(mx::Device::cpu));
mx::eval(idxh, wh);
const int32_t *ip = idxh.data<int32_t>();
const float *wp = wh.data<float>();
std::vector<int64_t> flat(static_cast<size_t>(T * k));
for (int64_t i = 0; i < T * k; i++) flat[static_cast<size_t>(i)] = ip[i];
mx::array routed(0.f);
const bool full = !prefill_ondemand() && io_->full_ready(li) && T * k > 8 &&
!io_->hot_ready(li);
if (full) {
io_->note_prefill_use(li, flat, (int)k,
li >= reg_.first_head_consumer && li <= reg_.last_head_consumer);
const auto &fw = io_->full_weights(li);
if ((int)fw.size() != 9) { err = "full-layer stack not loaded L" + std::to_string(li); return false; }
auto xe = mx::reshape(m_in, mkshape({1, T, 1, 1, H_})); // 1246 expand_dims((-2,-3))
auto iflat = mx::reshape(idx, mkshape({T * k}));
auto order = mx::argsort(iflat, -1); // _gather_sort(switch_layers.py:12-17)
auto inv_order = mx::argsort(order, -1);
auto xs = mx::flatten(xe, 0, -3); // [T,1,H]
auto tok_rows = mx::take(
xs, mx::floor_divide(order, mx::array((uint32_t)k, mx::uint32)), 0); // x[order//M]
auto local = mx::take(iflat, order, 0);
auto gq = [&](const mx::array &xx, int o) {
return mx::gather_qmm(xx, fw[o], fw[o + 1], fw[o + 2], std::nullopt, local, true, 64, 4,
"affine", true);
};
auto x_up = gq(tok_rows, 0);
auto x_gate = gq(tok_rows, 3);
auto yh = (x_gate * mx::sigmoid(x_gate)) * x_up; // _swiglu(1250;layer.py:57-58)
auto z = gq(yh, 6);
auto z_un = mx::take(z, inv_order, 0); // _scatter_unsort(1260)
routed = mx::squeeze(mx::reshape(z_un, mkshape({1, T, k, 1, H_})), -2);
auto &cx = exec_counters();
cx.qmm_calls.fetch_add(3, std::memory_order_relaxed);
cx.qmm_experts.fetch_add(3 * static_cast<uint64_t>(E), std::memory_order_relaxed);
cx.qmm_weight_bytes.fetch_add(
static_cast<uint64_t>(E) *
(2 * (moe_ffn_ * (H_ / 8) * 4 + 2 * moe_ffn_ * (H_ / group_) * 2) +
(H_ * (moe_ffn_ / 8) * 4 + 2 * H_ * (moe_ffn_ / group_) * 2)),
std::memory_order_relaxed);
}
if (!full) {
std::vector<int64_t> uniq = flat; // 1364 sorted(set(int(v) for v in flat))
std::sort(uniq.begin(), uniq.end());
uniq.erase(std::unique(uniq.begin(), uniq.end()), uniq.end());
std::vector<int32_t> local_h(static_cast<size_t>(T * k)); // 1370-1373 host remap
for (int64_t i = 0; i < T * k; i++)
local_h[static_cast<size_t>(i)] = static_cast<int32_t>(
std::lower_bound(uniq.begin(), uniq.end(), flat[static_cast<size_t>(i)]) - uniq.begin());
auto bundles = io_->get_bundles(li, uniq);
if (bundles.size() != uniq.size()) {
err = "expert load missing items L" + std::to_string(li);
return false;
}
exec_counters().leases.fetch_add(uniq.size(), std::memory_order_relaxed);
for (auto &[e, b] : bundles) step_bundles_.push_back(b);
std::vector<mx::array> wargs; // 1376-1380 wargs=core.stack(rows)
for (int p = 0; p < 9; p++) {
std::vector<mx::array> rows;
for (auto e : uniq) rows.push_back(bundles[e]->at(p));
wargs.push_back(mx::stack(rows));
}
auto local_arr = mx::array(local_h.begin(), mkshape({1, T, k}), mx::int32); // 1371 core.array(list)
const int F = static_cast<int>(moe_ffn_);
const int gsh = static_cast<int>(H_ / group_), gsf = static_cast<int>(moe_ffn_ / group_);
auto &cx = exec_counters();
cx.qmm_calls.fetch_add(3, std::memory_order_relaxed);
cx.qmm_experts.fetch_add(3 * static_cast<uint64_t>(uniq.size()), std::memory_order_relaxed);
cx.qmm_weight_bytes.fetch_add(
static_cast<uint64_t>(uniq.size()) *
(2 * (F * (H_ / 8) * 4 + 2 * F * gsh * 2) + (H_ * (F / 8) * 4 + 2 * H_ * gsf * 2)),
std::memory_order_relaxed);
if (T * k >= 64) {
auto flat_local = mx::reshape(local_arr, mkshape({T * k}));
auto order = mx::argsort(flat_local, -1); // _gather_sort: order=argsort(flat)
auto inv_order = mx::argsort(order, -1); // inv=argsort(order)
auto xe = mx::reshape(m_in, mkshape({1, T, 1, 1, H_})); // 1383 expand_dims((-2,-3)) [B,T,1,1,H]
auto xs = mx::flatten(xe, 0, -3); // 13-15 [T,1,H]
auto tok_rows = mx::take(
xs, mx::floor_divide(order, mx::array((uint32_t)k, mx::uint32)), 0);
auto local_sorted = mx::take(flat_local, order, 0); // indices[order]
std::vector<mx::array> sargs{tok_rows};
for (auto &wa : wargs) sargs.push_back(wa);
sargs.push_back(local_sorted);
auto z = moe_math_sorted_(sargs)[0];
auto z_un = mx::take(z, inv_order, 0);
routed = mx::squeeze(mx::reshape(z_un, mkshape({1, T, k, 1, H_})), -2); // unflatten+squeeze(-2)
} else {
std::vector<mx::array> args{m_in};
for (auto &wa : wargs) args.push_back(wa);
args.push_back(local_arr);
routed = moe_math_(args)[0];
}
}
auto out = mx::sum(routed * mx::astype(mx::expand_dims(w, -1), routed.dtype()),
std::vector<int>{-2}, false);
mx::array sh =
L.sh_down.apply(e0_silu(L.sh_gate.apply(m_in)) * L.sh_up.apply(m_in)); // BailingMLP:503
gt_cpu.stop();
x = h + out + sh; // :762
if (clip_ > 0)
x = mx::where(mx::isnan(x), mx::zeros_like(x),
mx::clip(x, mx::array(static_cast<float>(-clip_)),
mx::array(static_cast<float>(clip_))));
rec.layer = li;
rec.predicted = false;
{
std::vector<std::pair<int64_t, double>> lv;
for (int64_t j = 0; j < k; j++)
lv.push_back({ip[(T - 1) * k + j], wp[(T - 1) * k + j]});
std::sort(lv.begin(), lv.end());
for (auto &[e, s] : lv) {
rec.sel.inds.push_back(e);
rec.sel.scores.push_back(s);
}
}
if (full) {
mx::async_eval(x);
} else {
mx::eval(x);
step_bundles_.clear();
}
return true;
}
std::vector<float> mhost;
if (!to_host32(m_in, mhost, err)) return false; // [T,H]
e0n::TimeSink gt_cpu(e0n::timing_counters().gate_cpu_ns);
const int64_t E = reg_.num_experts;
const double *bias = L.gate_bias.empty() ? nullptr : L.gate_bias.data();
std::vector<Selection> sels(static_cast<size_t>(T));
bool predicted = false;
if (decode && st.pr.take_pred(li, sels[0])) {
predicted = true;
} else {
std::vector<double> lg;
gate_logits(mhost.data(), static_cast<int>(T), static_cast<int>(H_), L.gate_h.data(),
static_cast<int>(E), lg);
for (int64_t t = 0; t < T; t++)
sels[static_cast<size_t>(t)] = select_sigmoid_group(
lg.data() + t * E, bias, static_cast<int>(E), reg_.top_k, reg_.n_group,
reg_.topk_group, reg_.routed_scaling, reg_.norm_topk_prob);
}
rec.layer = li;
rec.predicted = predicted;
rec.sel = sels[static_cast<size_t>(T - 1)];
const bool owner = li >= reg_.first_owner && li <= reg_.last_owner;
if (owner) {
std::vector<int64_t> feat = rec.sel.inds;
if (predicted && reg_.head_feature_teacher) {
std::vector<double> lg;
gate_logits(mhost.data() + (T - 1) * H_, 1, static_cast<int>(H_), L.gate_h.data(),
static_cast<int>(E), lg);
feat = select_sigmoid_group(lg.data(), bias, static_cast<int>(E), reg_.top_k, reg_.n_group,
reg_.topk_group, reg_.routed_scaling, reg_.norm_topk_prob)
.inds;
}
std::vector<float> last_row(mhost.begin() + (T - 1) * H_, mhost.end());
st.pr.capture(li, std::move(last_row), std::move(feat));
if (prefill_seed && reg_.head_feature_teacher) {
std::vector<int64_t> prev;
if (T >= 2) {
std::vector<double> lg;
gate_logits(mhost.data() + (T - 2) * H_, 1, static_cast<int>(H_), L.gate_h.data(),
static_cast<int>(E), lg);
prev = select_sigmoid_group(lg.data(), bias, static_cast<int>(E), reg_.top_k,
reg_.n_group, reg_.topk_group, reg_.routed_scaling,
reg_.norm_topk_prob)
.inds;
}
st.pr.prev_exec[li] = std::move(prev);
}
}
gt_cpu.stop();
std::vector<int64_t> uni;
for (auto &s : sels)
for (auto e : s.inds)
if (uni.empty() || uni.back() != e) uni.push_back(e);
std::sort(uni.begin(), uni.end());
uni.erase(std::unique(uni.begin(), uni.end()), uni.end());
auto bundles = io_->get_bundles(li, uni);
exec_counters().leases.fetch_add(uni.size(), std::memory_order_relaxed);
std::vector<ExpertPtrs> ptrs(uni.size());
for (size_t i = 0; i < uni.size(); i++)
ptrs[i] = bundle_ptrs(*bundles[uni[i]]);
std::vector<float> routed(static_cast<size_t>(T) * H_, 0.f);
const int F = static_cast<int>(moe_ffn_);
const int gsh = static_cast<int>(H_ / group_), gsf = static_cast<int>(moe_ffn_ / group_);
{
for (size_t ei = 0; ei < uni.size(); ei++) {
int64_t e = uni[ei];
std::vector<int> toks;
for (int64_t t = 0; t < T; t++)
for (auto id : sels[static_cast<size_t>(t)].inds)
if (id == e) toks.push_back(static_cast<int>(t));
if (toks.empty()) continue;
std::vector<float> rows(toks.size() * H_);
for (size_t k = 0; k < toks.size(); k++)
std::memcpy(rows.data() + k * H_, mhost.data() + static_cast<size_t>(toks[k]) * H_,
H_ * 4);
std::vector<int64_t> one{0};
std::vector<float> yg, yu;
if (!gather_qmm_mlx(ptrs[ei].gw, ptrs[ei].gs, ptrs[ei].gb, 1, F, static_cast<int>(H_), gsh,
rows.data(), static_cast<int>(toks.size()), one, yg, err) ||
!gather_qmm_mlx(ptrs[ei].uw, ptrs[ei].us, ptrs[ei].ub, 1, F, static_cast<int>(H_), gsh,
rows.data(), static_cast<int>(toks.size()), one, yu, err))
return false;
std::vector<float> yh(toks.size() * moe_ffn_);
for (size_t k = 0; k < yh.size(); k++) yh[k] = silu_f(yg[k]) * yu[k];
std::vector<float> yd;
if (!gather_qmm_mlx(ptrs[ei].dw, ptrs[ei].ds, ptrs[ei].db, 1, static_cast<int>(H_), F, gsf,
yh.data(), static_cast<int>(toks.size()), one, yd, err))
return false;
for (size_t k = 0; k < toks.size(); k++) {
int t = toks[k];
double w = 0.0;
auto &s = sels[static_cast<size_t>(t)];
for (size_t j = 0; j < s.inds.size(); j++)
if (s.inds[j] == e) w = s.scores[j];
for (int64_t o = 0; o < H_; o++)
routed[static_cast<size_t>(t) * H_ + o] += static_cast<float>(w * yd[k * H_ + o]);
}
}
}
mx::array sh =
L.sh_down.apply(e0_silu(L.sh_gate.apply(m_in)) * L.sh_up.apply(m_in)); // [1,T,H]
std::vector<float> shost;
if (!to_host32(sh, shost, err)) return false;
for (int64_t t = 0; t < T; t++)
for (int64_t o = 0; o < H_; o++)
routed[static_cast<size_t>(t) * H_ + o] += shost[static_cast<size_t>(t) * H_ + o];
mx::array y_arr = mx::astype(wrap_f32(routed, mkshape({1, T, H_})), dt);
x = h + y_arr;
if (clip_ > 0) {
x = mx::where(mx::isnan(x), mx::zeros_like(x),
mx::clip(x, mx::array(static_cast<float>(-clip_)),
mx::array(static_cast<float>(clip_))));
}
E0_TIME(e0n::timing_counters().eval_ns);
mx::eval(x);
return true;
}
bool Net8b::stage_all_gpu(State8b &st, std::string &err, DecodeProbe *probe) {
const int64_t E = reg_.num_experts;
std::vector<int> metas;
std::vector<mx::array> ms, curs, prevs;
for (int o : pr_->owners()) {
const auto &m = m_in_cache_[static_cast<size_t>(o)];
if (m.ndim() < 3) continue;
metas.push_back(o);
mx::Shape s0 = m.shape(), s1 = m.shape();
for (auto &v : s0) v = 0;
s0[s0.size() - 2] = static_cast<mx::Shape::value_type>(m.shape(-2) - 1);
ms.push_back(mx::slice(m, s0, s1));
mx::array co(0.f); // 284 cur=pos-1 one-hot
if (!onehot_pos(last_topk_[static_cast<size_t>(o)], E, -1, co))
co = mx::zeros(mkshape({1, 1, E}), mx::float32);
curs.push_back(co);
mx::array po(0.f);
if (prev_topk_oh_[static_cast<size_t>(o)].ndim() > 0)
po = prev_topk_oh_[static_cast<size_t>(o)];
else if (!onehot_pos(last_topk_[static_cast<size_t>(o)], E, -2, po))
po = mx::zeros(mkshape({1, 1, E}), mx::float32); // 292-293
prevs.push_back(po);
}
if (metas.empty()) return true; // 185-186
const int64_t n = static_cast<int64_t>(metas.size());
auto s_ms = mx::stack(ms, 0), s_curs = mx::stack(curs, 0), s_prevs = mx::stack(prevs, 0);
auto feats = mx::astype(mx::concatenate({s_ms, s_curs, s_prevs}, -1), mx::float16); // 158-161
auto f2 = mx::reshape(feats, mkshape({n, H_ + 2 * E})); // 162
auto gel = [](const mx::array &x) {
return mx::array(0.5f, mx::float16) * x *
(mx::array(1.0f, mx::float16) +
mx::erf(x / mx::array(static_cast<float>(std::sqrt(2.0)), mx::float16)));
};
auto h1 = mx::einsum("ni,nij->nj", {f2, hw1_}); // 163
auto out = mx::einsum("ni,nij->nj", {f2, hwl_}) +
mx::einsum("ni,nij->nj", {gel(h1), hw2_}); // 165-166
auto logits_all = mx::reshape(out, mkshape({n, 1, 1, E})); // 167
mx::array inds_all(0.f), scores_all(0.f);
select_group_gpu(logits_all, nullptr, reg_.top_k, reg_.n_group, reg_.topk_group,
reg_.routed_scaling, reg_.norm_topk_prob, inds_all, scores_all); // _select 296-300
auto inds_h = mx::add(inds_all, mx::array((uint32_t)0, mx::uint32));
auto scores_h = mx::add(scores_all, mx::array(0.0f, mx::float32));
mx::eval(inds_h, scores_h); // 193 ONE eval
exec_counters().host_syncs.fetch_add(1, std::memory_order_relaxed);
const int k = static_cast<int>(reg_.top_k);
const uint32_t *ip = inds_h.data<uint32_t>(); // 194 ONE tolist
const float *sp = scores_h.data<float>();
for (int64_t i = 0; i < n; i++) {
const int o = metas[i], consumer = static_cast<int>(o + 1);
Selection s;
std::vector<int64_t> eids;
for (int j = 0; j < k; j++) eids.push_back(ip[i * k + j]);
std::sort(eids.begin(), eids.end());
eids.erase(std::unique(eids.begin(), eids.end()), eids.end());
for (auto e : eids) {
double v = 0.0;
for (int j = 0; j < k; j++)
if (ip[i * k + j] == static_cast<uint32_t>(e)) v = sp[i * k + j];
s.inds.push_back(e);
s.scores.push_back(v);
}
if (consumer <= reg_.last_head_consumer) {
st.pr.pred[static_cast<size_t>(consumer)] = s;
st.pr.pred_avail[static_cast<size_t>(consumer)] = true;
io_->stage_experts(consumer, s.inds);
mx::Shape a0 = logits_all.shape(), a1 = logits_all.shape();
for (auto &v : a0) v = 0;
a0[0] = static_cast<mx::Shape::value_type>(i);
a1[0] = static_cast<mx::Shape::value_type>(i + 1);
pg_logits_[static_cast<size_t>(consumer)] =
mx::squeeze(mx::slice(logits_all, a0, a1), 0);
}
prev_topk_oh_[static_cast<size_t>(o)] = curs[i];
}
st.pr.step++;
if (probe) {
auto lg32 = mx::astype(logits_all, mx::float32);
mx::eval(lg32);
exec_counters().host_syncs.fetch_add(1, std::memory_order_relaxed);
const float *lp = lg32.data<float>();
probe->owners.clear();
probe->m_in.clear();
probe->logits.clear();
for (int64_t i = 0; i < n; i++) {
probe->owners.push_back(metas[i]);
probe->logits.emplace_back(lp + i * E, lp + (i + 1) * E);
std::vector<float> mv;
auto r32 = mx::astype(mx::reshape(ms[i], mkshape({H_})), mx::float32);
if (!to_host32(r32, mv, err)) return false;
probe->m_in.push_back(std::move(mv));
}
}
return true;
}
void Net8b::reset_graph_caches() const {
const size_t NL = layers_.size();
m_in_cache_.assign(NL, mx::array(0.f));
last_topk_.assign(NL, mx::array(0.f));
prev_topk_oh_.assign(NL, mx::array(0.f));
pg_logits_.assign(NL, mx::array(0.f));
for (auto &L : layers_)
if (L.has_moe) io_->reset_layer(static_cast<int>(&L - layers_.data())); // layer.py:1429-1441
step_bundles_.clear();
}
void Net8b::prefetch(const Step &plan) {
std::map<int, std::vector<int64_t>> by_layer;
for (auto &[li, e] : plan.experts) by_layer[li].push_back(e);
for (auto &[li, es] : by_layer) io_->prefetch(li, es);
}
void Net8b::shard_warm(const std::vector<std::string> &shard_names, bool seq) {
for (auto &s : shard_names) {
io_->shard_advise_whole(s);
if (seq) io_->shard_seq_read(s);
}
}
bool Net8b::prefill(const std::vector<int32_t> &tokens, State8b &st,
std::vector<float> &logits_last, std::vector<RouteRec> &recs,
std::string &err, PrefillProbe *probe) {
if (exec_mode() == ExecMode::Batched)
return prefill_batched_(tokens, st, logits_last, recs, err, probe);
try {
std::vector<float> xh;
if (!embed_lookup(embed_, tokens, H_, V_, xh, err)) return false;
const int64_t T = static_cast<int64_t>(tokens.size());
if (probe) probe->embed_last.assign(xh.begin() + (T - 1) * H_, xh.end());
mx::array x = mx::astype(wrap_f32(xh, mkshape({1, T, H_})), mx::bfloat16);
recs.clear();
for (size_t li = 0; li < layers_.size(); li++) {
RouteRec rec;
if (!layer_fwd(layers_[li], static_cast<int>(li), x, st, false, true, rec, err))
return false;
if (probe) {
mx::array tail = mx::astype(
mx::reshape(mx::slice(x, mkshape({0, T - 1, 0}), mkshape({1, T, H_})),
mkshape({1, H_})),
mx::float32);
std::vector<float> hv;
if (!to_host32(tail, hv, err)) return false;
probe->per_block.push_back(std::move(hv));
}
if (layers_[li].has_moe) recs.push_back(std::move(rec));
}
mx::array xs = e0_rms_norm(
x, mx::astype(wrap_f32(final_norm_w_, mkshape({H_})), mx::bfloat16), eps_);
mx::array last = mx::reshape(mx::slice(xs, mkshape({0, T - 1, 0}), mkshape({1, T, H_})),
mkshape({1, H_}));
if (!to_host32(lm_head_.apply(last), logits_last, err)) return false; // [V]
st.position += T;
if (!st.pr.run_step_boundary(*pr_, nullptr, err)) return false;
} catch (const std::exception &ex) {
err = std::string("prefill: ") + ex.what();
return false;
}
return true;
}
bool Net8b::prefill_batched_(const std::vector<int32_t> &tokens, State8b &st,
std::vector<float> &logits_last, std::vector<RouteRec> &recs,
std::string &err, PrefillProbe *probe) {
try {
const int64_t total = static_cast<int64_t>(tokens.size());
const int64_t chunk = reg_.prefill_chunk > 0 ? reg_.prefill_chunk : total; // base.py:39(wrapper :71=2048)
recs.clear();
std::vector<float> lg;
for (int64_t start = 0; start < total; start += chunk) {
const int64_t n = std::min(chunk, total - start);
const bool last = start + n >= total;
step_bundles_.clear();
if (!forward_chunk_batched_(tokens.data() + start, n, st, lg, last, recs, err,
last ? probe : nullptr))
return false;
st.position += n; // base.py:83
}
if (!prefill_end_(st, err)) return false;
logits_last = std::move(lg);
return true;
} catch (const std::exception &ex) {
err = std::string("prefill: ") + ex.what();
return false;
}
}
bool Net8b::forward_chunk_batched_(const int32_t *tok, int64_t T, State8b &st,
std::vector<float> &logits_out, bool to_host,
std::vector<RouteRec> &recs, std::string &err,
PrefillProbe *probe) {
std::vector<float> xh;
if (!embed_lookup(embed_, std::vector<int32_t>(tok, tok + T), H_, V_, xh, err)) return false;
if (probe) probe->embed_last.assign(xh.begin() + (T - 1) * H_, xh.end());
mx::array x = mx::astype(wrap_f32(xh, mkshape({1, T, H_})), mx::bfloat16);
const bool cb = T > 1 && !prefill_ondemand() &&
(reg_.full_layer_prefill || reg_.prefill_hot > 0);
std::vector<RouteRec> local_recs;
for (size_t li = 0; li < layers_.size(); li++) {
if (cb) prefill_before_layer(static_cast<int>(li));
RouteRec rec;
if (!layer_fwd(layers_[li], static_cast<int>(li), x, st, false, true, rec, err))
return false;
if (probe) {
mx::array tail = mx::astype(
mx::reshape(mx::slice(x, mkshape({0, T - 1, 0}), mkshape({1, T, H_})),
mkshape({1, H_})),
mx::float32);
std::vector<float> hv;
if (!to_host32(tail, hv, err)) return false;
probe->per_block.push_back(std::move(hv));
}
if (layers_[li].has_moe) local_recs.push_back(std::move(rec));
}
mx::array xs = e0_rms_norm(
x, mx::astype(wrap_f32(final_norm_w_, mkshape({H_})), mx::bfloat16), eps_);
mx::array lastrow =
mx::reshape(mx::slice(xs, mkshape({0, T - 1, 0}), mkshape({1, T, H_})), mkshape({1, H_}));
mx::array lg = lm_head_.apply(lastrow); // ling.py:220
if (to_host) {
if (!to_host32(lg, logits_out, err)) return false;
recs = std::move(local_recs);
} else {
mx::eval(lg);
}
return true;
}
void Net8b::prefill_before_layer(int li) {
io_->clear_full_layer(li - 1);
if (li >= 0 && li < (int)layers_.size() && layers_[size_t(li)].has_moe)
io_->load_full_layer(li);
}
bool Net8b::prefill_end_(State8b &st, std::string &err) {
for (size_t li = 0; li < layers_.size(); li++) io_->clear_full_layer(static_cast<int>(li)); // :278-279
return stage_all_gpu(st, err, nullptr);
}
bool Net8b::step(int32_t token, State8b &st, std::vector<float> &logits,
std::vector<RouteRec> &recs, std::string &err, DecodeProbe *probe) {
try {
step_bundles_.clear();
std::vector<float> xh;
if (!embed_lookup(embed_, {token}, H_, V_, xh, err)) return false;
mx::array x = mx::astype(wrap_f32(xh, mkshape({1, 1, H_})), mx::bfloat16);
recs.clear();
for (size_t li = 0; li < layers_.size(); li++) {
RouteRec rec;
if (!layer_fwd(layers_[li], static_cast<int>(li), x, st, true, false, rec, err))
return false;
if (layers_[li].has_moe) recs.push_back(std::move(rec));
}
mx::array xs = e0_rms_norm(
x, mx::astype(wrap_f32(final_norm_w_, mkshape({H_})), mx::bfloat16), eps_);
if (!to_host32(lm_head_.apply(mx::reshape(xs, mkshape({1, H_}))), logits, err)) return false;
st.position += 1;
if (exec_mode() == ExecMode::Batched) {
if (!stage_all_gpu(st, err, probe)) return false;
} else if (probe) {
probe->owners.clear(); probe->m_in.clear(); probe->logits.clear();
for (int h = 0; h < pr_->n_heads(); h++) {
int own = pr_->owner(h);
if (!st.pr.cap_hidden[own].empty()) {
probe->owners.push_back(own);
probe->m_in.push_back(st.pr.cap_hidden[own]);
}
}
std::vector<std::vector<double>> diag;
if (!st.pr.run_step_boundary(*pr_, &diag, err)) return false;
probe->logits = std::move(diag);
} else if (!st.pr.run_step_boundary(*pr_, nullptr, err)) return false;
} catch (const std::exception &ex) {
err = std::string("step: ") + ex.what();
return false;
}
return true;
}
} // namespace e0n