Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
54.1 KB
Raw Blame
#include "net35.hpp"

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <cstdlib>
#include <functional>
#include <numeric>  // iota (batched selk)
#include <set>

#include "dense.hpp"
#include "qgemm.hpp"

namespace e0n {
namespace mx = mlx::core;

namespace {

using S = mx::Shape;

namespace {
inline uint16_t f_to_bf16(float x) {
  uint32_t u;
  std::memcpy(&u, &x, 4);
  u += 0x7FFFu + ((u >> 16) & 1u);
  return static_cast<uint16_t>(u >> 16);
}
inline void bf_round(std::vector<float> &v) {
  for (float &x : v) {
    uint32_t u = static_cast<uint32_t>(f_to_bf16(x)) << 16;
    std::memcpy(&x, &u, 4);
  }
}
}  // namespace

mx::array first_k(const mx::array &a, int64_t k) {
  mx::Shape s1 = a.shape();
  mx::Shape s0 = a.shape();
  for (auto &v : s0) v = 0;
  s1[s1.size() - 1] = static_cast<mx::Shape::value_type>(k);
  return mx::slice(a, s0, s1);
}
// heads.py select_from_logits(44-51) ≡ moe/routing.py select_from_logits(13-24):
void select_softmax_gpu(const mx::array &logits, int64_t E, int64_t k, bool norm,
                        mx::array &idx, mx::array &w) {
  (void)E;  // shape is carried by logits; retained in the call signature for router symmetry.
  auto gates = mx::softmax(logits, std::vector<int>{-1}, true /*precise*/);  // heads.py:47
  idx = first_k(mx::argpartition(-gates, static_cast<int>(k - 1), -1), k);  // 48
  w = mx::take_along_axis(gates, idx, -1);                                // 49
  if (norm) w = w / mx::sum(w, std::vector<int>{-1}, true);               // 50(norm_topk_prob)
}

mx::array topk_onehot(const mx::array &inds, int64_t E) {
  auto eq = mx::equal(mx::expand_dims(inds, -1), mx::arange(0, static_cast<int>(E)));
  return mx::astype(mx::sum(eq, std::vector<int>{-2}, false), mx::float32);
}

mx::array gelu_erf16(const mx::array &x) {
  return mx::array(0.5f, mx::float16) * x *
         (mx::array(1.0f, mx::float16) +
          mx::erf(x / mx::array(static_cast<float>(std::sqrt(2.0)), mx::float16)));
}

std::vector<mx::array> moe_math_body(const std::vector<mx::array> &a) {
  auto xe = mx::reshape(a[0], mkshape({a[0].shape(0), a[0].shape(1), 1, 1, a[0].shape(2)}));  // layer.py:159
  auto gq = [&](const mx::array &xx, int o) {
    return mx::gather_qmm(xx, a[o], a[o + 1], a[o + 2], std::nullopt, a[10], true, 64, 4,
                          "affine", false);
  };
  auto x_up = gq(xe, 1);
  auto x_gate = gq(xe, 4);
  auto yh = (x_gate * mx::sigmoid(x_gate)) * x_up;  // _swiglu(x_up, x_gate)(layer.py:57-58,169)
  auto z = gq(yh, 7);
  return {mx::squeeze(z, -2)};                      // 172
}

std::vector<mx::array> moe_math_sorted_body(const std::vector<mx::array> &a) {
  auto gq = [&](const mx::array &xx, int o) {
    return mx::gather_qmm(xx, a[o], a[o + 1], a[o + 2], std::nullopt, a[10], true, 64, 4,
                          "affine", true);
  };
  auto x_up = gq(a[0], 1);
  auto x_gate = gq(a[0], 4);
  auto yh = (x_gate * mx::sigmoid(x_gate)) * x_up;             // _swiglu
  return {gq(yh, 7)};                                          // z=[T*k,1,H]
}

ExpertPtrs bundle_ptrs(const Bundle &b) {
  ExpertPtrs p;
  p.uw = b.at(0).data<uint32_t>();
  p.us = reinterpret_cast<const uint16_t *>(b.at(1).data<mx::bfloat16_t>());
  p.ub = reinterpret_cast<const uint16_t *>(b.at(2).data<mx::bfloat16_t>());
  p.gw = b.at(3).data<uint32_t>();
  p.gs = reinterpret_cast<const uint16_t *>(b.at(4).data<mx::bfloat16_t>());
  p.gb = reinterpret_cast<const uint16_t *>(b.at(5).data<mx::bfloat16_t>());
  p.dw = b.at(6).data<uint32_t>();
  p.ds = reinterpret_cast<const uint16_t *>(b.at(7).data<mx::bfloat16_t>());
  p.db = reinterpret_cast<const uint16_t *>(b.at(8).data<mx::bfloat16_t>());
  return p;
}

}  // namespace

std::unique_ptr<Net35> Net35::create(std::shared_ptr<const ModelFiles> files,
                                     std::shared_ptr<const ExpertMap> emap, std::string &err,
                                     std::shared_ptr<const lora::Bundle> lo) {
  std::unique_ptr<Net35> n(new Net35());
  n->files_ = files;
  n->emap_ = emap;
  n->lo_ = std::move(lo);
  if (!tier_routing("edge0-35b", n->reg_)) { err = "registry missing edge0-35b"; return nullptr; }
  int64_t cfg_layers = 0;
  if (files->config_int("num_hidden_layers", cfg_layers) && cfg_layers != n->reg_.num_layers) {
    err = "config layer count does not match registry";
    return nullptr;
  }
  auto ci = [&](const char *k, int64_t def, int64_t &o) {
    int64_t v;
    o = files->config_int(k, v) ? v : def;
  };
  auto cd = [&](const char *k, double def, double &o) {
    double v;
    o = files->config_double(k, v) ? v : def;
  };
  ci("hidden_size", 2048, n->H_);
  ci("vocab_size", 248320, n->V_);
  ci("moe_intermediate_size", 512, n->moe_ffn_);
  ci("shared_expert_intermediate_size", 512, n->sh_ffn_);
  ci("linear_num_key_heads", 16, n->hk_);
  ci("linear_num_value_heads", 32, n->hv_);
  ci("linear_key_head_dim", 128, n->dk_);
  ci("linear_value_head_dim", 128, n->dv_);
  ci("linear_conv_kernel_dim", 4, n->conv_k_);
  ci("num_attention_heads", 16, n->nq_);
  ci("num_key_value_heads", 2, n->nkv_);
  ci("head_dim", 256, n->hd_);
  int64_t interval = 4;
  ci("full_attention_interval", 4, interval);
  n->full_interval_ = static_cast<double>(interval);
  cd("rms_norm_eps", 1e-6, n->eps_);
  cd("rope_theta", 100000.0, n->rope_base_);
  cd("partial_rotary_factor", 0.25, n->partial_rot_);
  bool og = true;
  if (!files->config_bool("attn_output_gate", og)) og = true;
  n->attn_gate_ = og;

  const int64_t keyd = n->hk_ * n->dk_, vald = n->hv_ * n->dv_;
  n->conv_dim_ = 2 * keyd + vald;
  {
    static const std::string kGdnSource = R"gdn(
        auto n = thread_position_in_grid.z;
        auto b_idx = n / Hv;
        auto hv_idx = n % Hv;
        auto hk_idx = hv_idx / (Hv / Hk);
        constexpr int n_per_t = Dk / 32;

        // q, k: [B, T, Hk, Dk]
        auto q_ = q + b_idx * T * Hk * Dk + hk_idx * Dk;
        auto k_ = k + b_idx * T * Hk * Dk + hk_idx * Dk;

        // v, y: [B, T, Hv, Dv]
        auto v_ = v + b_idx * T * Hv * Dv + hv_idx * Dv;
        y += b_idx * T * Hv * Dv + hv_idx * Dv;

        auto dk_idx = thread_position_in_threadgroup.x;
        auto dv_idx = thread_position_in_grid.y;

        // state_in, state_out: [B, Hv, Dv, Dk]
        auto i_state = state_in + (n * Dv + dv_idx) * Dk;
        auto o_state = state_out + (n * Dv + dv_idx) * Dk;

        float state[n_per_t];
        for (int i = 0; i < n_per_t; ++i) {
          auto s_idx = n_per_t * dk_idx + i;
          state[i] = static_cast<float>(i_state[s_idx]);
        }

        // g: [B, T, Hv]
        auto g_ = g + b_idx * T * Hv;
        auto beta_ = beta + b_idx * T * Hv;

        for (int t = 0; t < T; ++t) {
          if (true) {
            float kv_mem = 0.0f;
            for (int i = 0; i < n_per_t; ++i) {
              auto s_idx = n_per_t * dk_idx + i;
              state[i] = state[i] * g_[hv_idx];
              kv_mem += state[i] * k_[s_idx];
            }
            kv_mem = simd_sum(kv_mem);

            auto delta = (v_[dv_idx] - kv_mem) * beta_[hv_idx];

            float out = 0.0f;
            for (int i = 0; i < n_per_t; ++i) {
              auto s_idx = n_per_t * dk_idx + i;
              state[i] = state[i] + k_[s_idx] * delta;
              out += state[i] * q_[s_idx];
            }
            out = simd_sum(out);
            if (thread_index_in_simdgroup == 0) {
              y[dv_idx] = static_cast<InT>(out);
            }
          }
          // Increment data pointers to next time step
          q_ += Hk * Dk;
          k_ += Hk * Dk;
          v_ += Hv * Dv;
          y += Hv * Dv;
          g_ += Hv;
          beta_ += Hv;
        }
        for (int i = 0; i < n_per_t; ++i) {
          auto s_idx = n_per_t * dk_idx + i;
          o_state[s_idx] = static_cast<InT>(state[i]);
        }
    )gdn";
    n->gdn_kernel_ = mx::fast::metal_kernel(
        "gated_delta_step", {"q", "k", "v", "g", "beta", "state_in", "T"}, {"y", "state_out"},
        kGdnSource);
  }

  n->layers_.resize(static_cast<size_t>(n->reg_.num_layers));
  const std::string M = "language_model.model.layers.";
  for (int64_t li = 0; li < n->reg_.num_layers; li++) {
    Layer &L = n->layers_[static_cast<size_t>(li)];
    const std::string P = M + std::to_string(li);
    L.is_linear = ((li + 1) % interval) != 0;
    auto q = [&](const std::string &name, QuantLin &dst) {
      if (!dst.load(*files, P + name, err)) { err += " @" + std::to_string(li); return false; }
      n->dense_bytes_ += dst.wb.size() + dst.sb.size() + dst.bb.size();
      return true;
    };
    auto vec = [&](const std::string &name, std::vector<float> &dstv) {
      DenseF32 d;
      if (!read_dense_f32(*files, P + name, d, err)) {
        err += " @" + std::to_string(li);
        return false;
      }
      dstv = std::move(d.v);
      n->dense_bytes_ += dstv.size() * 4;
      return true;
    };
    if (!vec(".input_layernorm.weight", L.in_ln_w) ||
        !vec(".post_attention_layernorm.weight", L.post_ln_w))
      return nullptr;
    if (L.is_linear) {
      if (!q(".linear_attn.in_proj_qkv.weight", L.in_proj_qkv) ||
          !q(".linear_attn.in_proj_z.weight", L.in_proj_z) ||
          !q(".linear_attn.in_proj_b.weight", L.in_proj_b) ||
          !q(".linear_attn.in_proj_a.weight", L.in_proj_a) ||
          !q(".linear_attn.out_proj.weight", L.out_proj) ||
          !vec(".linear_attn.A_log", L.A_log) || !vec(".linear_attn.dt_bias", L.dt_bias) ||
          !vec(".linear_attn.norm.weight", L.gnorm_w))
        return nullptr;
      if (static_cast<int64_t>(L.A_log.size()) != n->hv_ ||
          static_cast<int64_t>(L.dt_bias.size()) != n->hv_ ||
          static_cast<int64_t>(L.gnorm_w.size()) != n->dv_) {
        err = "GDN parameter vector length mismatch";
        return nullptr;
      }
      if (!files->read_tensor(P + ".linear_attn.conv1d.weight", L.conv_wb, err)) return nullptr;
      L.conv_w = mx::array(L.conv_wb.data(), mkshape({n->conv_dim_, n->conv_k_, 1}),
                           mx::bfloat16, [](void *) {});
      n->dense_bytes_ += L.conv_wb.size();
    } else {
      if (!q(".self_attn.q_proj.weight", L.q_proj) || !q(".self_attn.k_proj.weight", L.k_proj) ||
          !q(".self_attn.v_proj.weight", L.v_proj) || !q(".self_attn.o_proj.weight", L.o_proj) ||
          !vec(".self_attn.q_norm.weight", L.qn_w) || !vec(".self_attn.k_norm.weight", L.kn_w))
        return nullptr;
    }
    DenseF32 g;
    int64_t gbits = 4, ggs = 64;
    const std::string gw = P + ".mlp.gate.weight";
    if (!files->config_quant_for(gw, gbits, ggs)) { err = "gate has no quantization config"; return nullptr; }
    if (!read_dense_dequant(*files, gw, gbits, ggs, g, err)) return nullptr;
    if (g.rows != n->reg_.num_experts || g.cols != n->H_) {
      err = "gate shape mismatch";
      return nullptr;
    }
    L.gate_h = std::move(g.v);
    n->dense_bytes_ += L.gate_h.size() * 4;
    if (!L.gate_q.load(*files, gw, err)) { err += " @gate_q"; return nullptr; }
    n->dense_bytes_ += L.gate_q.wb.size() + L.gate_q.sb.size() + L.gate_q.bb.size();
    if (!q(".mlp.shared_expert.gate_proj.weight", L.sh_gate) ||
        !q(".mlp.shared_expert.up_proj.weight", L.sh_up) ||
        !q(".mlp.shared_expert.down_proj.weight", L.sh_down) ||
        !q(".mlp.shared_expert_gate.weight", L.sh_egate))
      return nullptr;
  }
  if (!n->embed_.load(*files, "language_model.model.embed_tokens.weight", err)) return nullptr;
  if (!n->lm_head_.load(*files, "language_model.lm_head.weight", err)) return nullptr;
  n->dense_bytes_ += n->embed_.wb.size() + n->embed_.sb.size() + n->embed_.bb.size() +
                     n->lm_head_.wb.size() + n->lm_head_.sb.size() + n->lm_head_.bb.size();
  {
    DenseF32 f;
    if (!read_dense_f32(*files, "language_model.model.norm.weight", f, err)) return nullptr;
    n->final_norm_w_ = std::move(f.v);
  }
  n->pr_ = std::make_unique<Prerouter>();
  if (!Prerouter::load(*files, n->reg_, n->H_, *n->pr_, err)) return nullptr;
  {
    const int64_t NL2 = n->reg_.num_layers;
    n->io_ = std::make_unique<ExpertIo>(files.get(), 64, 48, 8, 4);
    for (int64_t li = 0; li < NL2; li++) {
      ExpertIo::LayerCfg c;
      c.layer = int(li);
      c.prefix = emap->layer_prefix(int(li));
      c.E = n->reg_.num_experts;
      c.H = n->H_;
      c.F = n->moe_ffn_;
      c.group = 64;
      c.staged_n = int(n->reg_.top_k);
      c.incr_mode = true;
      c.incr_writeback = true;
      c.warm_willneed = true;
      c.prefill_hot = (int)n->reg_.prefill_hot;
      c.full_layer_prefill = n->reg_.full_layer_prefill;
      n->io_->add_layer(c);
    }
  }
  if (exec_mode() == ExecMode::Batched) {
    const char *cl = getenv("MLX_CACHE_LIMIT_MB");
    mx::set_cache_limit(static_cast<size_t>(cl ? atoi(cl) : 256) << 20);
    const int64_t NL = n->reg_.num_layers;
    n->m_in_cache_.assign(NL, mx::array(0.f));
    n->oh_cur_.assign(NL, mx::array(0.f));
    n->oh_prev_.assign(NL, mx::array(0.f));
    n->pred_inds_gpu_.assign(NL, mx::array(0.f));
    n->pred_scores_gpu_.assign(NL, mx::array(0.f));
    const int64_t nH = n->pr_->n_heads(), f_in = n->pr_->stacked_f_in(),
                  prh = n->pr_->stacked_prh(), hE = n->pr_->stacked_E();
    if (nH > 0) {
      auto wrap16 = [&](const std::vector<float> &v, std::initializer_list<int64_t> dims) {
        return mx::astype(mx::array((void *)v.data(), mkshape(dims), mx::float32, [](void *) {}),
                          mx::float16);
      };
      n->hw1_ = wrap16(n->pr_->stacked_fc1(), {nH, f_in, prh});
      n->hw2_ = wrap16(n->pr_->stacked_fc2(), {nH, prh, hE});
      n->hwl_ = wrap16(n->pr_->stacked_lin(), {nH, f_in, hE});
      mx::eval(n->hw1_, n->hw2_, n->hwl_);
    }
    // @core.compile(layer.py:157;staged_k4 use_compile=True,options.py:129).
    try {
      auto fn = mx::compile(&moe_math_body);
      const int64_t F = n->moe_ffn_;
      std::vector<mx::array> d = {
          mx::zeros(mkshape({1, 1, n->H_}), mx::bfloat16),
          mx::zeros(mkshape({2, F, n->H_ / 8}), mx::uint32),
          mx::zeros(mkshape({2, F, n->H_ / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, F, n->H_ / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, F, n->H_ / 8}), mx::uint32),
          mx::zeros(mkshape({2, F, n->H_ / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, F, n->H_ / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, n->H_, F / 8}), mx::uint32),
          mx::zeros(mkshape({2, n->H_, F / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, n->H_, F / 64}), mx::bfloat16),
          mx::zeros(mkshape({1, 1, 2}), mx::int32)};
      auto out = fn(d);
      mx::eval(out[0]);
      n->moe_math_ = std::move(fn);
    } catch (const std::exception &ex) {
      std::fprintf(stderr, "edge0-35b: mlx compile unavailable (%s), MoE math falling back to eager\n",
                   ex.what());
      n->moe_math_ = &moe_math_body;
    }
    try {
      auto fn = mx::compile(&moe_math_sorted_body);
      const int64_t F = n->moe_ffn_;
      std::vector<mx::array> d = {
          mx::zeros(mkshape({16, 1, n->H_}), mx::bfloat16),
          mx::zeros(mkshape({2, F, n->H_ / 8}), mx::uint32),
          mx::zeros(mkshape({2, F, n->H_ / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, F, n->H_ / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, F, n->H_ / 8}), mx::uint32),
          mx::zeros(mkshape({2, F, n->H_ / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, F, n->H_ / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, n->H_, F / 8}), mx::uint32),
          mx::zeros(mkshape({2, n->H_, F / 64}), mx::bfloat16),
          mx::zeros(mkshape({2, n->H_, F / 64}), mx::bfloat16),
          mx::zeros(mkshape({16}), mx::int32)};
      auto out = fn(d);
      mx::eval(out[0]);
      n->moe_math_sorted_ = std::move(fn);
    } catch (const std::exception &ex) {
      std::fprintf(stderr, "edge0-35b: mlx compile unavailable (%s), sorted MoE math falling back to eager\n",
                   ex.what());
      n->moe_math_sorted_ = &moe_math_sorted_body;
    }
  }
  if (n->lo_) {
    const int nl = n->reg_.num_layers;
    const int full = nl / 4, lin = nl - full;
    const int want = 5 * lin + 4 * full + 3 * nl;
    if (n->lo_->n_bound() != want) {
      err = "LoRA bound count " + std::to_string(n->lo_->n_bound()) + " != tier structural count " +
            std::to_string(want);
      return nullptr;
    }
    n->dense_bytes_ += static_cast<uint64_t>(n->lo_->bytes());
  }
  return n;
}

void Net35::init_state(State35 &st) const {
  st.pr.init(reg_);
  const size_t NL = layers_.size();
  st.conv.assign(NL, mx::array(0.f));
  st.recur.assign(NL, mx::array(0.f));
  st.kc.assign(NL, mx::array(0.f));
  st.vc.assign(NL, mx::array(0.f));
  st.is_linear.assign(NL, 0);
  for (size_t i = 0; i < NL; i++) st.is_linear[i] = layers_[i].is_linear ? 1 : 0;
  st.position = 0;
  st.started = true;
  reset_graph_caches();
}

void Net35::reset_state(State35 &st) const {
  st.pr.reset();
  st.conv.assign(layers_.size(), mx::array(0.f));
  st.recur.assign(layers_.size(), mx::array(0.f));
  st.kc.assign(layers_.size(), mx::array(0.f));
  st.vc.assign(layers_.size(), mx::array(0.f));
  st.position = 0;
  reset_graph_caches();
}

bool Net35::layer_fwd(Layer &L, int li, mx::array &x, State35 &st, bool decode, RouteRec &rec,
                      mx::array *m_in_out, std::string &err) {
  const int64_t T = x.shape(1);
  const auto dt = x.dtype();
  const int64_t keyd = hk_ * dk_, vald = hv_ * dv_;
  auto norm_w = [&](const std::vector<float> &v) {
    return mx::astype(wrap_f32(v, mkshape({static_cast<int64_t>(v.size())})), dt);
  };
  mx::array xn = e0_rms_norm(x, norm_w(L.in_ln_w), eps_);

  mx::array r{0.f};
  if (L.is_linear) {
    const auto *lo = lo_.get();
    using LS = e0n::lora::Slot35;
    mx::array qkv = e0n::lora::apply_lora(lo, L.in_proj_qkv, xn, li, LS::S35_IN_QKV);  // [1,T,conv_dim]
    mx::array z = mx::reshape(e0n::lora::apply_lora(lo, L.in_proj_z, xn, li, LS::S35_IN_Z),
                              mkshape({1, T, hv_, dv_}));
    mx::array b = e0n::lora::apply_lora(lo, L.in_proj_b, xn, li, LS::S35_IN_B);  // [1,T,hv]
    mx::array a = e0n::lora::apply_lora(lo, L.in_proj_a, xn, li, LS::S35_IN_A);
    if (st.conv[li].ndim() == 0)
      st.conv[li] = mx::zeros(mkshape({1, conv_k_ - 1, conv_dim_}), dt);
    mx::array cat = mx::concatenate({st.conv[li], qkv}, 1);  // [1, K-1+T, C]
    st.conv[li] = mx::slice(cat, mkshape({0, T, 0}), mkshape({1, T + conv_k_ - 1, conv_dim_}));
    mx::array convout = e0_silu(mx::conv1d(
        cat, L.conv_w, 1, 0, 1, static_cast<int>(conv_dim_)));  // [1,T,C]
    auto parts = mx::split(convout, S{static_cast<int32_t>(keyd), static_cast<int32_t>(2 * keyd)},
                           -1);
    mx::array qf = mx::reshape(parts[0], mkshape({1, T, hk_, dk_}));
    mx::array kf = mx::reshape(parts[1], mkshape({1, T, hk_, dk_}));
    mx::array vf = mx::reshape(parts[2], mkshape({1, T, hv_, dv_}));
    float inv = 1.0f / std::sqrt(static_cast<float>(dk_));
    auto scalar_bf = [&](float v) { return mx::astype(mx::array(v), dt); };
    qf = scalar_bf(inv * inv) * e0_rms_norm(qf, std::nullopt, 1e-6);
    kf = scalar_bf(inv) * e0_rms_norm(kf, std::nullopt, 1e-6);
    mx::array beta = mx::sigmoid(b);
    mx::array alog = wrap_f32(L.A_log, mkshape({hv_}));
    mx::array dtb = mx::astype(wrap_f32(L.dt_bias, mkshape({hv_})), dt);  // bf16
    mx::array xab = a + dtb;
    mx::array sp = mx::log1p(mx::exp(-mx::abs(xab))) +
                   mx::maximum(xab, mx::astype(mx::array(0.f), dt));
    mx::array g = mx::astype(mx::exp(-mx::exp(alog) * mx::astype(sp, mx::float32)), dt);
    if (st.recur[li].ndim() == 0)
      st.recur[li] = mx::zeros(mkshape({1, hv_, dv_, dk_}), dt);  // gated_delta_update:275-278
    auto gouts = gdn_kernel_(
        {qf, kf, vf, g, beta, st.recur[li], mx::array((int)T, mx::int32)},
        {mkshape({1, T, hv_, dv_}), st.recur[li].shape()},
        {dt, dt},
        {32, static_cast<int>(dv_), static_cast<int>(hv_)},  // grid=(32, Dv, B*Hv),B=1
        {32, 4, 1},                                          // threadgroup(gated_delta.py:206)
        {{"InT", dt},
         {"Dk", static_cast<int>(dk_)},
         {"Dv", static_cast<int>(dv_)},
         {"Hk", static_cast<int>(hk_)},
         {"Hv", static_cast<int>(hv_)}},
        std::nullopt, false, {});
    mx::array y_arr = gouts[0];
    st.recur[li] = gouts[1];
    mx::array ng = e0_rms_norm(y_arr, norm_w(L.gnorm_w), eps_);
    mx::array gated = e0_swiglu_precise(dt, z, ng);  // _precise_swiglu(out, z, x):silu(z)*x
    r = e0n::lora::apply_lora(lo_.get(), L.out_proj,
                              mx::reshape(gated, mkshape({1, T, vald})), li,
                              e0n::lora::S35_OUT_PROJ);
  } else {
    mx::array qo = e0n::lora::apply_lora(lo_.get(), L.q_proj, xn, li, e0n::lora::S35_Q);
    auto qs = mx::split(mx::reshape(qo, mkshape({1, T, nq_, hd_ * 2})),
                        S{static_cast<int32_t>(hd_)}, -1);
    mx::array qg = qs[0], gate = mx::reshape(qs[1], mkshape({1, T, nq_ * hd_}));
    mx::array q = mx::transpose(e0_rms_norm(qg, norm_w(L.qn_w), eps_), {0, 2, 1, 3});
    mx::array k = mx::transpose(
        e0_rms_norm(mx::reshape(e0n::lora::apply_lora(lo_.get(), L.k_proj, xn, li,
                                                   e0n::lora::S35_K),
                                   mkshape({1, T, nkv_, hd_})),
                    norm_w(L.kn_w), eps_),
        {0, 2, 1, 3});
    mx::array v = mx::transpose(mx::reshape(e0n::lora::apply_lora(lo_.get(), L.v_proj, xn, li,
                                                                        e0n::lora::S35_V),
                                                        mkshape({1, T, nkv_, hd_})),
                                {0, 2, 1, 3});
    const int rdim = static_cast<int>(hd_ * partial_rot_);
    q = mx::fast::rope(q, rdim, false, static_cast<float>(rope_base_), 1.0f,
                       static_cast<int>(st.position));
    k = mx::fast::rope(k, rdim, false, static_cast<float>(rope_base_), 1.0f,
                       static_cast<int>(st.position));
    if (st.kc[li].ndim() == 0) {
      st.kc[li] = k;
      st.vc[li] = v;
    } else {
      st.kc[li] = mx::concatenate({st.kc[li], k}, 2);
      st.vc[li] = mx::concatenate({st.vc[li], v}, 2);
    }
    mx::array o = mx::fast::scaled_dot_product_attention(
        q, st.kc[li], st.vc[li], 1.0f / std::sqrt(static_cast<float>(hd_)), "causal");
    o = mx::reshape(mx::transpose(o, {0, 2, 1, 3}), mkshape({1, T, nq_ * hd_}));
    if (attn_gate_) o = o * mx::sigmoid(gate);
    r = e0n::lora::apply_lora(lo_.get(), L.o_proj, o, li, e0n::lora::S35_O);
  }

  mx::array h = x + r;
  mx::array m_in = e0_rms_norm(h, norm_w(L.post_ln_w), eps_);
  if (m_in_out) *m_in_out = m_in;
  if (exec_mode() == ExecMode::Batched && decode && T == 1) {
    const int64_t E = reg_.num_experts, k = reg_.top_k;
    io_->note_call(li);
    const bool owner = li >= reg_.first_owner && li <= reg_.last_owner;   // 6..38
    const bool consumer = li >= reg_.first_head_consumer && li <= reg_.last_head_consumer;  // 7..38
    mx::array idx(0.f), w(0.f), routed(0.f);
    rec.layer = li;
    if (consumer && pred_inds_gpu_[static_cast<size_t>(li)].ndim() > 0) {
      idx = pred_inds_gpu_[static_cast<size_t>(li)];
      w = pred_scores_gpu_[static_cast<size_t>(li)];
      rec.predicted = true;
      rec.sel = st.pr.pred[static_cast<size_t>(li)];
      st.pr.pred_avail[static_cast<size_t>(li)] = false;
      io_->wait_staged(li);
      auto local = mx::take(io_->incr_slot_table(li), idx);  // layer.py:1338 take(_incr_slot_table, indices)
      std::vector<mx::array> args{m_in};
      for (auto &a : io_->incr_tensors(li)) args.push_back(a);
      args.push_back(local);
      auto &cx = exec_counters();
      cx.qmm_calls.fetch_add(3, std::memory_order_relaxed);
      cx.qmm_experts.fetch_add(3 * static_cast<uint64_t>(k), std::memory_order_relaxed);
      cx.qmm_weight_bytes.fetch_add(
          static_cast<uint64_t>(k) *
              (2 * (moe_ffn_ * (H_ / 8) * 4 + 2 * moe_ffn_ * (H_ / 64) * 2) +
               (H_ * (moe_ffn_ / 8) * 4 + 2 * H_ * (moe_ffn_ / 64) * 2)),
          std::memory_order_relaxed);
      routed = moe_math_(args)[0];  // layer.py:1341
    } else {
      const int F = static_cast<int>(moe_ffn_);
      const int gsh = static_cast<int>(H_ / 64), gsf = static_cast<int>(moe_ffn_ / 64);
      auto gl = L.gate_q.apply(m_in);
      select_softmax_gpu(gl, E, k, reg_.norm_topk_prob, idx, w);
      mx::eval(idx, w);
      exec_counters().host_syncs.fetch_add(1, std::memory_order_relaxed);
      auto idxh = mx::copy(mx::astype(idx, mx::int32), mx::default_stream(mx::Device::cpu));
      auto wh = mx::copy(mx::astype(w, mx::float32), mx::default_stream(mx::Device::cpu));
      mx::eval(idxh, wh);
      const int32_t *ip = idxh.data<int32_t>();
      const float *wp = wh.data<float>();
      std::vector<int64_t> flat(ip, ip + k);
      std::vector<int64_t> uniq = flat;
      std::sort(uniq.begin(), uniq.end());
      uniq.erase(std::unique(uniq.begin(), uniq.end()), uniq.end());
      std::vector<int32_t> local_h(flat.size());
      for (size_t j = 0; j < flat.size(); j++)
        local_h[j] = static_cast<int32_t>(
            std::lower_bound(uniq.begin(), uniq.end(), flat[j]) - uniq.begin());
      auto bundles = io_->get_bundles(li, uniq);
      if (bundles.size() != uniq.size()) {
        err = "expert load missing L" + std::to_string(li);
        return false;
      }
      exec_counters().leases.fetch_add(uniq.size(), std::memory_order_relaxed);
      for (auto &[e, b] : bundles) step_bundles_.push_back(b);
      std::vector<mx::array> args{m_in};
      for (int p = 0; p < 9; p++) {
        std::vector<mx::array> rows;
        for (auto e : uniq) rows.push_back(bundles[e]->at(p));
        args.push_back(mx::stack(rows));
      }
      args.push_back(mx::array(local_h.begin(), mkshape({1, 1, k}), mx::int32));  // 1371
      auto &cx = exec_counters();
      cx.qmm_calls.fetch_add(3, std::memory_order_relaxed);
      cx.qmm_experts.fetch_add(3 * static_cast<uint64_t>(uniq.size()), std::memory_order_relaxed);
      cx.qmm_weight_bytes.fetch_add(
          static_cast<uint64_t>(uniq.size()) *
              (2 * (F * (H_ / 8) * 4 + 2 * F * gsh * 2) + (H_ * (F / 8) * 4 + 2 * H_ * gsf * 2)),
          std::memory_order_relaxed);
      routed = moe_math_(args)[0];
      rec.predicted = false;
      rec.sel.inds = uniq;
      rec.sel.scores.clear();
      for (int64_t e : uniq) {
        double s = 0.0;
        for (int64_t j = 0; j < k; j++)
          if (flat[j] == e) s = wp[j];
        rec.sel.scores.push_back(s);
      }
    }
    if (owner) {
      m_in_cache_[static_cast<size_t>(li)] = m_in;
      oh_cur_[static_cast<size_t>(li)] = topk_onehot(idx, E);
    }
    // mix+shared(install.py:190-194):(y*scores[...,None]).sum(-2) + sigmoid(egate(x))*shared(x)
    auto out = mx::sum(routed * mx::expand_dims(w, -1), std::vector<int>{-2}, false);
    auto shg = e0n::lora::apply_lora(lo_.get(), L.sh_gate, m_in, li, e0n::lora::S35_SH_GATE);
    auto shu = e0n::lora::apply_lora(lo_.get(), L.sh_up, m_in, li, e0n::lora::S35_SH_UP);
    mx::array sh =
        e0n::lora::apply_lora(lo_.get(), L.sh_down, e0_silu(shg) * shu, li, e0n::lora::S35_SH_DOWN);
    mx::array eg = mx::sigmoid(L.sh_egate.apply(m_in));
    x = h + out + sh * eg;
    if (clip_ > 0)
      x = mx::where(mx::isnan(x), mx::zeros_like(x),
                    mx::clip(x, mx::array(static_cast<float>(-clip_)),
                             mx::array(static_cast<float>(clip_))));
    return true;
  }
  if (exec_mode() == ExecMode::Batched) {
    const int64_t E = reg_.num_experts, k = reg_.top_k;
    io_->note_call(li);
    mx::array idx(0.f), w(0.f);
    rec.layer = li;
    rec.predicted = false;
    {
      auto gl = L.gate_q.apply(m_in);
      select_softmax_gpu(gl, E, k, reg_.norm_topk_prob, idx, w);  // qwen3_next.py:335-341
    }
    mx::eval(idx, w);
    exec_counters().host_syncs.fetch_add(1, std::memory_order_relaxed);
    auto idxh = mx::copy(mx::astype(idx, mx::int32), mx::default_stream(mx::Device::cpu));
    auto wh = mx::copy(mx::astype(w, mx::float32), mx::default_stream(mx::Device::cpu));
    mx::eval(idxh, wh);
    const int32_t *ip = idxh.data<int32_t>();
    const float *wp = wh.data<float>();
    std::vector<int64_t> flat(static_cast<size_t>(T * k));
    for (int64_t i = 0; i < T * k; i++) flat[static_cast<size_t>(i)] = ip[i];
    mx::array routed(0.f);
    const bool hot = !prefill_ondemand() && T * k > 8 && io_->hot_ready(li) &&
                     !io_->full_ready(li);  // __call__ guard(layer.py:1121-1122)
    if (hot) {
      io_->note_prefill_use(li, flat, (int)k,
                            li >= reg_.first_head_consumer && li <= reg_.last_head_consumer);
      if (!moe_hot_branch_(li, m_in, T, flat, routed, err)) return false;
    }
    if (!hot) {
      std::vector<int64_t> uniq = flat;                     // 1364 sorted(set(...))
      std::sort(uniq.begin(), uniq.end());
      uniq.erase(std::unique(uniq.begin(), uniq.end()), uniq.end());
      std::vector<int32_t> local_h(static_cast<size_t>(T * k));  // 1370-1373 host remap
      for (int64_t i = 0; i < T * k; i++)
        local_h[static_cast<size_t>(i)] = static_cast<int32_t>(
            std::lower_bound(uniq.begin(), uniq.end(), flat[static_cast<size_t>(i)]) - uniq.begin());
      auto bundles = io_->get_bundles(li, uniq);            // 1368
      if (bundles.size() != uniq.size()) {
        err = "expert load missing L" + std::to_string(li);
        return false;
      }
      exec_counters().leases.fetch_add(uniq.size(), std::memory_order_relaxed);
      for (auto &[e, b] : bundles) step_bundles_.push_back(b);
      std::vector<mx::array> wargs;                         // 1376-1380 core.stack(rows)
      for (int p = 0; p < 9; p++) {
        std::vector<mx::array> rows;
        for (auto e : uniq) rows.push_back(bundles[e]->at(p));
        wargs.push_back(mx::stack(rows));
      }
      auto local_arr = mx::array(local_h.begin(), mkshape({1, T, k}), mx::int32);  // 1371
      const int F = static_cast<int>(moe_ffn_);
      auto &cx = exec_counters();
      cx.qmm_calls.fetch_add(3, std::memory_order_relaxed);
      cx.qmm_experts.fetch_add(3 * static_cast<uint64_t>(uniq.size()), std::memory_order_relaxed);
      cx.qmm_weight_bytes.fetch_add(
          static_cast<uint64_t>(uniq.size()) *
              (2 * (F * (H_ / 8) * 4 + 2 * F * (H_ / 64) * 2) +
               (H_ * (F / 8) * 4 + 2 * H_ * (F / 64) * 2)),
          std::memory_order_relaxed);
      if (T * k >= 64) {                            // 1384 do_sort = local2d.size >= 64
        auto flat_local = mx::reshape(local_arr, mkshape({T * k}));
        auto order = mx::argsort(flat_local, -1);   // _gather_sort(switch_layers.py:12-17)
        auto inv_order = mx::argsort(order, -1);
        auto xe = mx::reshape(m_in, mkshape({1, T, 1, 1, H_}));  // 1383 expand_dims((-2,-3))
        auto xs = mx::flatten(xe, 0, -3);           // [T,1,H]
        auto tok_rows = mx::take(
            xs, mx::floor_divide(order, mx::array((uint32_t)k, mx::uint32)), 0);  // x[order//M]
        auto local_sorted = mx::take(flat_local, order, 0);  // indices[order]
        std::vector<mx::array> sargs{tok_rows};
        for (auto &wa : wargs) sargs.push_back(wa);
        sargs.push_back(local_sorted);
        auto z = moe_math_sorted_(sargs)[0];        // [T*k,1,H]
        auto z_un = mx::take(z, inv_order, 0);
        routed = mx::squeeze(mx::reshape(z_un, mkshape({1, T, k, 1, H_})), -2);
      } else {
        std::vector<mx::array> args{m_in};
        for (auto &wa : wargs) args.push_back(wa);
        args.push_back(local_arr);
        routed = moe_math_(args)[0];
      }
    }
    // mix+shared(install.py:190-194):(y*scores[...,None]).sum(-2) + sigmoid(egate(x))*shared(x)
    auto out = mx::sum(routed * mx::expand_dims(w, -1), std::vector<int>{-2}, false);
    auto shg = e0n::lora::apply_lora(lo_.get(), L.sh_gate, m_in, li, e0n::lora::S35_SH_GATE);
    auto shu = e0n::lora::apply_lora(lo_.get(), L.sh_up, m_in, li, e0n::lora::S35_SH_UP);
    mx::array sh =
        e0n::lora::apply_lora(lo_.get(), L.sh_down, e0_silu(shg) * shu, li, e0n::lora::S35_SH_DOWN);
    mx::array eg = mx::sigmoid(L.sh_egate.apply(m_in));
    x = h + out + sh * eg;
    if (clip_ > 0)
      x = mx::where(mx::isnan(x), mx::zeros_like(x),
                    mx::clip(x, mx::array(static_cast<float>(-clip_)),
                             mx::array(static_cast<float>(clip_))));
    {
      std::vector<std::pair<int64_t, double>> lv;
      for (int64_t j = 0; j < k; j++)
        lv.push_back({ip[(T - 1) * k + j], wp[(T - 1) * k + j]});
      std::sort(lv.begin(), lv.end());
      for (auto &[e, s] : lv) {
        rec.sel.inds.push_back(e);
        rec.sel.scores.push_back(s);
      }
    }
    if (!hot) {
      mx::eval(x);
      step_bundles_.clear();
    }
    return true;
  }

  std::vector<float> mhost;
  if (!to_host32(m_in, mhost, err)) return false;  // [T,H]

  const int64_t E = reg_.num_experts;
  std::vector<Selection> sels(static_cast<size_t>(T));
  bool predicted = false;
  if (decode && st.pr.take_pred(li, sels[0])) {
    predicted = true;
  } else {
    std::vector<double> lg;
    {
      mx::array glh = L.gate_q.apply(m_in);
      std::vector<float> glf;
      if (!to_host32(glh, glf, err)) return false;
      lg.assign(glf.begin(), glf.end());
    }
    for (int64_t t = 0; t < T; t++)
      sels[static_cast<size_t>(t)] = select_softmax_topk(
          lg.data() + t * E, static_cast<int>(E), reg_.top_k, reg_.norm_topk_prob);
  }
  rec.layer = li;
  rec.predicted = predicted;
  rec.sel = sels[static_cast<size_t>(T - 1)];
  std::vector<float> last_row(mhost.begin() + (T - 1) * H_, mhost.end());
  st.pr.capture(li, std::move(last_row), rec.sel.inds);

  std::vector<int64_t> uni;
  for (auto &s : sels)
    for (auto e : s.inds)
      if (uni.empty() || uni.back() != e) uni.push_back(e);
  std::sort(uni.begin(), uni.end());
  uni.erase(std::unique(uni.begin(), uni.end()), uni.end());
  auto bundles = io_->get_bundles(li, uni);
  exec_counters().leases.fetch_add(uni.size(), std::memory_order_relaxed);
  std::vector<ExpertPtrs> ptrs(uni.size());
  for (size_t i = 0; i < uni.size(); i++)
    ptrs[i] = bundle_ptrs(*bundles[uni[i]]);
  std::vector<float> routed(static_cast<size_t>(T) * H_, 0.f);
  const int F = static_cast<int>(moe_ffn_);
  {
  for (size_t ei = 0; ei < uni.size(); ei++) {
    int64_t e = uni[ei];
    std::vector<int> toks;
    for (int64_t t = 0; t < T; t++)
      for (auto id : sels[static_cast<size_t>(t)].inds)
        if (id == e) toks.push_back(static_cast<int>(t));
    if (toks.empty()) continue;
    std::vector<float> rows(toks.size() * H_);
    for (size_t k = 0; k < toks.size(); k++)
      std::memcpy(rows.data() + k * H_, mhost.data() + static_cast<size_t>(toks[k]) * H_,
                  H_ * 4);
    std::vector<int64_t> one{0};
    std::vector<float> yg, yu;
    if (!gather_qmm_mlx(ptrs[ei].gw, ptrs[ei].gs, ptrs[ei].gb, 1, F, static_cast<int>(H_),
                        static_cast<int>(H_ / 64), rows.data(), static_cast<int>(toks.size()), one,
                        yg, err) ||
        !gather_qmm_mlx(ptrs[ei].uw, ptrs[ei].us, ptrs[ei].ub, 1, F, static_cast<int>(H_),
                        static_cast<int>(H_ / 64), rows.data(), static_cast<int>(toks.size()), one,
                        yu, err))
      return false;
    std::vector<float> yh(toks.size() * moe_ffn_);
    {
      bf_round(yg);
      bf_round(yu);
      auto bf = [](float v) {
        uint32_t u; std::memcpy(&u, &v, 4);
        u += 0x7FFFu + ((u >> 16) & 1u); u >>= 16; u <<= 16;
        float o; std::memcpy(&o, &u, 4); return o;
      };
      for (size_t k = 0; k < yh.size(); k++) {
        float g = yg[k];
        float sg = bf(1.0f / (1.0f + std::exp(-g)));
        float sl = bf(g * sg);
        yh[k] = bf(sl * yu[k]);
      }
    }
    std::vector<float> yd;
    if (!gather_qmm_mlx(ptrs[ei].dw, ptrs[ei].ds, ptrs[ei].db, 1, static_cast<int>(H_), F,
                        static_cast<int>(moe_ffn_ / 64), yh.data(),
                        static_cast<int>(toks.size()), one, yd, err))
      return false;
    bf_round(yd);
    for (size_t k = 0; k < toks.size(); k++) {
      int t = toks[k];
      double w = 0.0;
      auto &s = sels[static_cast<size_t>(t)];
      for (size_t j = 0; j < s.inds.size(); j++)
        if (s.inds[j] == e) w = s.scores[j];
      for (int64_t o = 0; o < H_; o++)
        routed[static_cast<size_t>(t) * H_ + o] += static_cast<float>(w * yd[k * H_ + o]);
    }
  }
  }

  auto shg = e0n::lora::apply_lora(lo_.get(), L.sh_gate, m_in, li, e0n::lora::S35_SH_GATE);
  auto shu = e0n::lora::apply_lora(lo_.get(), L.sh_up, m_in, li, e0n::lora::S35_SH_UP);
  mx::array sh = e0n::lora::apply_lora(
      lo_.get(), L.sh_down, e0_silu(shg) * shu, li, e0n::lora::S35_SH_DOWN);  // [1,T,H]
  mx::array eg = mx::sigmoid(L.sh_egate.apply(m_in));                      // [1,T,1]
  std::vector<float> shost;
  if (!to_host32(sh * eg, shost, err)) return false;
  for (int64_t t = 0; t < T; t++)
    for (int64_t o = 0; o < H_; o++)
      routed[static_cast<size_t>(t) * H_ + o] += shost[static_cast<size_t>(t) * H_ + o];
  mx::array y_arr = mx::astype(mx::reshape(wrap_f32(routed, mkshape({1, T, H_})),
                                           mkshape({1, T, H_})),
                               dt);
  x = h + y_arr;
  if (clip_ > 0) {
    x = mx::where(mx::isnan(x), mx::zeros_like(x),
                  mx::clip(x, mx::array(static_cast<float>(-clip_)),
                           mx::array(static_cast<float>(clip_))));
  }
  mx::eval(x);
  return true;
}

bool Net35::stage_all_gpu(State35 &st, std::string &err, DecodeProbe *probe) {
  const int64_t E = reg_.num_experts;
  std::vector<int> metas;
  std::vector<mx::array> ms, curs, prevs;
  for (int o : pr_->owners()) {
    if (m_in_cache_[static_cast<size_t>(o)].ndim() < 3 ||
        oh_cur_[static_cast<size_t>(o)].ndim() == 0)
      continue;
    metas.push_back(o);
    const auto &m = m_in_cache_[static_cast<size_t>(o)];
    mx::Shape s0 = m.shape(), s1 = m.shape();
    for (auto &v : s0) v = 0;
    s0[s0.size() - 2] = static_cast<mx::Shape::value_type>(m.shape(-2) - 1);
    ms.push_back(mx::slice(m, s0, s1));
    curs.push_back(oh_cur_[static_cast<size_t>(o)]);  // oh = executed(install:183-188)
    mx::array po(0.f);
    if (oh_prev_[static_cast<size_t>(o)].ndim() > 0)
      po = oh_prev_[static_cast<size_t>(o)];
    else
      po = mx::zeros(mkshape({1, 1, E}), mx::float32);
    prevs.push_back(po);
  }
  if (metas.empty()) return true;
  const int64_t n = static_cast<int64_t>(metas.size());
  auto feats = mx::astype(
      mx::concatenate({mx::stack(ms, 0), mx::stack(curs, 0), mx::stack(prevs, 0)}, -1),
      mx::float16);                            // stager.py:158-161
  auto f2 = mx::reshape(feats, mkshape({n, H_ + 2 * E}));  // 162
  auto h1 = mx::einsum("ni,nij->nj", {f2, hw1_});          // 163
  auto act = gelu_erf16(h1);
  auto lin = mx::einsum("ni,nij->nj", {f2, hwl_});
  auto fc2 = mx::einsum("ni,nij->nj", {act, hw2_});
  auto out = lin + fc2;                                      // 164-166
  auto logits_all = mx::reshape(out, mkshape({n, 1, 1, E}));    // 167
  mx::array inds_all(0.f), scores_all(0.f);
  select_softmax_gpu(logits_all, E, reg_.top_k, reg_.norm_topk_prob, inds_all, scores_all);  // _select:241-242
  // `first_k` is a strided view over the E-wide row. Materialize both outputs before
  // the single host read; reading data() from the view as if it were [n,k] walks into
  // the next row and corrupts the staged expert set (same fix as net8b).
  auto inds_h = mx::add(inds_all, mx::array((uint32_t)0, mx::uint32));
  auto scores_h = mx::add(scores_all, mx::array(0.0f, mx::float32));
  mx::eval(inds_h, scores_h);                  // stager:193 ONE eval
  exec_counters().host_syncs.fetch_add(1, std::memory_order_relaxed);
  const int k = static_cast<int>(reg_.top_k);
  const uint32_t *ip = inds_h.data<uint32_t>();  // 194 ONE tolist
  const float *sp = scores_h.data<float>();
  for (int64_t i = 0; i < n; i++) {
    const int o = metas[i], consumer = o + 1;  // 196-197
    Selection s;                                                     // _store:244-252
    std::vector<int64_t> eids;
    for (int j = 0; j < k; j++) eids.push_back(ip[i * k + j]);
    std::sort(eids.begin(), eids.end());
    eids.erase(std::unique(eids.begin(), eids.end()), eids.end());
    for (auto e : eids) {
      double v = 0.0;
      for (int j = 0; j < k; j++)
        if (ip[i * k + j] == static_cast<uint32_t>(e)) v = sp[i * k + j];
      s.inds.push_back(e);
      s.scores.push_back(v);
    }
    if (consumer >= reg_.first_head_consumer && consumer <= reg_.last_head_consumer) {
      st.pr.pred[static_cast<size_t>(consumer)] = s;
      st.pr.pred_avail[static_cast<size_t>(consumer)] = true;
      io_->stage_experts(consumer, s.inds);
      mx::Shape a0 = inds_all.shape(), a1 = inds_all.shape();  // pred_inds[p]/pred_scores[p]
      for (auto &v : a0) v = 0;
      a0[0] = static_cast<mx::Shape::value_type>(i);
      a1[0] = static_cast<mx::Shape::value_type>(i + 1);
      pred_inds_gpu_[static_cast<size_t>(consumer)] = mx::squeeze(mx::slice(inds_all, a0, a1), 0);     // [1,1,k]
      pred_scores_gpu_[static_cast<size_t>(consumer)] = mx::squeeze(mx::slice(scores_all, a0, a1), 0);
    }
    oh_prev_[static_cast<size_t>(o)] = curs[i];
  }
  st.pr.step++;
  if (probe) {
    auto lg32 = mx::astype(logits_all, mx::float32);
    mx::eval(lg32);
    exec_counters().host_syncs.fetch_add(1, std::memory_order_relaxed);
    const float *lp = lg32.data<float>();
    probe->owners.clear();
    probe->m_in.clear();
    probe->logits.clear();
    for (int64_t i = 0; i < n; i++) {
      probe->owners.push_back(metas[i]);
      probe->logits.emplace_back(lp + i * E, lp + (i + 1) * E);
      std::vector<float> mv;
      auto r32 = mx::astype(mx::reshape(ms[i], mkshape({H_})), mx::float32);
      if (!to_host32(r32, mv, err)) return false;
      probe->m_in.push_back(std::move(mv));
    }
  }
  return true;
}

void Net35::reset_graph_caches() const {
  const size_t NL = layers_.size();
  m_in_cache_.assign(NL, mx::array(0.f));
  oh_cur_.assign(NL, mx::array(0.f));
  oh_prev_.assign(NL, mx::array(0.f));
  pred_inds_gpu_.assign(NL, mx::array(0.f));
  pred_scores_gpu_.assign(NL, mx::array(0.f));
  for (size_t li = 0; li < layers_.size(); li++) io_->reset_layer(static_cast<int>(li));
  step_bundles_.clear();
}

void Net35::prefetch(const Step &plan) {
  if (pr_ && !pr_->owners().empty()) return;  // qwen.py:195(pg_stager is not None)
  std::map<int, std::vector<int64_t>> by_layer;
  for (auto &[li, e] : plan.experts) by_layer[li].push_back(e);
  for (auto &[li, es] : by_layer) io_->prefetch(li, es);
}

void Net35::shard_warm(const std::vector<std::string> &shard_names, bool seq) {
  for (auto &sn : shard_names) {
    io_->shard_advise_whole(sn);
    if (seq) io_->shard_seq_read(sn);
  }
}

bool Net35::prefill(const std::vector<int32_t> &tokens, State35 &st,
                    std::vector<float> &logits_last, std::vector<RouteRec> &recs,
                    std::string &err, PrefillProbe *probe) {
  if (exec_mode() == ExecMode::Batched)
    return prefill_batched_(tokens, st, logits_last, recs, err, probe);
  try {
    std::vector<float> xh;
    if (!embed_lookup(embed_, tokens, H_, V_, xh, err)) return false;
    const int64_t T = static_cast<int64_t>(tokens.size());
    if (probe) probe->embed_last.assign(xh.begin() + (T - 1) * H_, xh.end());
    mx::array x = mx::astype(mx::reshape(wrap_f32(xh, mkshape({1, T, H_})), mkshape({1, T, H_})),
                             mx::bfloat16);
    recs.clear();
    for (size_t li = 0; li < layers_.size(); li++) {
      RouteRec rec;
      if (!layer_fwd(layers_[li], static_cast<int>(li), x, st, false, rec, nullptr, err))
        return false;
      if (probe) {
        mx::array tail = mx::astype(
            mx::reshape(mx::slice(x, mkshape({0, T - 1, 0}), mkshape({1, T, H_})),
                        mkshape({1, H_})),
            mx::float32);
        std::vector<float> hv;
        if (!to_host32(tail, hv, err)) return false;
        probe->per_block.push_back(std::move(hv));
      }
      recs.push_back(std::move(rec));
    }
    mx::array xs = e0_rms_norm(
        x, mx::astype(wrap_f32(final_norm_w_, mkshape({H_})), mx::bfloat16), eps_);
    mx::array last = mx::reshape(mx::slice(xs, mkshape({0, T - 1, 0}), mkshape({1, T, H_})),
                                 mkshape({1, H_}));
    if (!to_host32(lm_head_.apply(last), logits_last, err)) return false;  // [V]
    st.position += T;
  } catch (const std::exception &ex) {
    err = std::string("prefill: ") + ex.what();
    return false;
  }
  return true;
}

bool Net35::prefill_batched_(const std::vector<int32_t> &tokens, State35 &st,
                             std::vector<float> &logits_last, std::vector<RouteRec> &recs,
                             std::string &err, PrefillProbe *probe) {
  try {
    const int64_t total = static_cast<int64_t>(tokens.size());
    const int64_t chunk = reg_.prefill_chunk > 0 ? reg_.prefill_chunk : total;  // base.py:39
    recs.clear();
    std::vector<float> lg;
    for (int64_t start = 0; start < total; start += chunk) {  // base.py:80-84
      const int64_t n = std::min(chunk, total - start);
      const bool last = start + n >= total;
      step_bundles_.clear();
      if (!forward_chunk_batched_(tokens.data() + start, n, st, lg, last, recs, err,
                                  last ? probe : nullptr))
        return false;
      st.position += n;  // base.py:83 self.pos += len(chunk)
      if (!stage_all_gpu(st, err, nullptr)) return false;
    }
    prefill_end_(st);
    logits_last = std::move(lg);
    return true;
  } catch (const std::exception &ex) {
    err = std::string("prefill: ") + ex.what();
    return false;
  }
}

bool Net35::forward_chunk_batched_(const int32_t *tok, int64_t T, State35 &st,
                                   std::vector<float> &logits_out, bool to_host,
                                   std::vector<RouteRec> &recs, std::string &err,
                                   PrefillProbe *probe) {
  std::vector<float> xh;
  if (!embed_lookup(embed_, std::vector<int32_t>(tok, tok + T), H_, V_, xh, err)) return false;
  if (probe) probe->embed_last.assign(xh.begin() + (T - 1) * H_, xh.end());
  mx::array x = mx::astype(mx::reshape(wrap_f32(xh, mkshape({1, T, H_})), mkshape({1, T, H_})),
                           mx::bfloat16);
  const bool cb = T > 1 && reg_.prefill_hot > 0 && !prefill_ondemand();
  std::vector<RouteRec> local_recs;
  for (size_t li = 0; li < layers_.size(); li++) {
    if (cb) prefill_before_layer(static_cast<int>(li));  // qwen3_5.py:285-287 before_layer_cb
    RouteRec rec;
    if (!layer_fwd(layers_[li], static_cast<int>(li), x, st, false, rec, nullptr, err))
      return false;
    if (probe) {
      mx::array tail = mx::astype(
          mx::reshape(mx::slice(x, mkshape({0, T - 1, 0}), mkshape({1, T, H_})),
                      mkshape({1, H_})),
          mx::float32);
      std::vector<float> hv;
      if (!to_host32(tail, hv, err)) return false;
      probe->per_block.push_back(std::move(hv));
    }
    local_recs.push_back(std::move(rec));
  }
  mx::array xs = e0_rms_norm(
      x, mx::astype(wrap_f32(final_norm_w_, mkshape({H_})), mx::bfloat16), eps_);
  mx::array lastrow =
      mx::reshape(mx::slice(xs, mkshape({0, T - 1, 0}), mkshape({1, T, H_})), mkshape({1, H_}));
  mx::array lg = lm_head_.apply(lastrow);  // [1,V](qwen.py:177)
  if (to_host) {
    if (!to_host32(lg, logits_out, err)) return false;
    recs = std::move(local_recs);
  } else {
    mx::eval(lg);
  }
  return true;
}

void Net35::prefill_before_layer(int li) {
  const int w = std::max(1, reg_.hot_window);  // hooks.py:31
  const int ahead = std::max(1, w / 2);        // :32
  io_->clear_full_layer(li - 1);
  if (reg_.prefill_hot) {                       // :40
    const int nL = static_cast<int>(layers_.size());
    for (int lj = li; lj < std::min(li + ahead + 1, nL); lj++) {
      io_->load_hot_layer(lj, reg_.prefill_hot);
      io_->materialize_hot(lj);
    }
    for (int lj = 0; lj < std::max(0, li - (w - ahead)); lj++)
      io_->dematerialize_hot(lj);
    return;
  }
}

// prefill_end_ = qwen.py:233-250 `_prefill_end`.
void Net35::prefill_end_(State35 &st) {
  for (size_t li = 0; li < layers_.size(); li++) io_->clear_full_layer(static_cast<int>(li));
  for (size_t li = 0; li < layers_.size(); li++) io_->refresh_hot_pins(static_cast<int>(li));
  for (int li = reg_.first_head_consumer; li <= reg_.last_head_consumer; li++)
    io_->prefetch_from_prefill(li);
  (void)st;
}

bool Net35::moe_hot_branch_(int li, const mx::array &m_in, int64_t T,
                            const std::vector<int64_t> &flat, mx::array &routed,
                            std::string &err) {
  const int64_t k = reg_.top_k, TK = static_cast<int64_t>(flat.size());
  const auto &key = io_->hot_key(li);         // _hot_key(1132)
  const auto &wst = io_->hot_weights(li);     // 9×[n+1,...](1131)
  if ((int)wst.size() != 9) { err = "hot window not materialized L" + std::to_string(li); return false; }
  std::map<int64_t, int32_t> pos;             // :1132 pos={e:i}
  for (size_t j = 0; j < key.size(); j++) pos[key[j]] = (int32_t)j;
  const int32_t n_rows = (int32_t)key.size();  // :1133
  std::vector<int32_t> g(TK);
  std::vector<int64_t> miss_rows;              // :1143
  for (int64_t i = 0; i < TK; i++) {
    auto it = pos.find(flat[i]);
    if (it == pos.end()) { g[i] = n_rows; miss_rows.push_back(i); }
    else g[i] = it->second;
  }
  auto remapped = mx::array(g.begin(), mkshape({T, k}), mx::int32);  // :1151-1152
  auto x_flat = mx::reshape(m_in, mkshape({T, H_}));                 // :1136
  auto xe = mx::reshape(x_flat, mkshape({T, 1, 1, H_}));             // :1153 expand_dims((-2,-3))
  auto flatr = mx::reshape(remapped, mkshape({TK}));
  auto order = mx::argsort(flatr, -1);                               // _gather_sort 1154
  auto inv_order = mx::argsort(order, -1);
  auto xs = mx::flatten(xe, 0, -3);                                  // [T,1,H]
  auto tok_rows = mx::take(xs, mx::floor_divide(order, mx::array((uint32_t)k, mx::uint32)), 0);
  auto local = mx::take(flatr, order, 0);                            // indices[order]
  const int F = static_cast<int>(moe_ffn_);
  auto gq = [&](const mx::array &xx, int o) {  // _gather_gate_up/down unfused+sorted(1095-1104,1156-1161)
    return mx::gather_qmm(xx, wst[o], wst[o + 1], wst[o + 2], std::nullopt, local, true, 64, 4,
                          "affine", true);
  };
  auto x_up = gq(tok_rows, 0);
  auto x_gate = gq(tok_rows, 3);
  auto yh = (x_gate * mx::sigmoid(x_gate)) * x_up;                   // _swiglu(1157;layer.py:57-58)
  auto z = mx::gather_qmm(yh, wst[6], wst[7], wst[8], std::nullopt, local, true, 64, 4,
                          "affine", true);
  auto out = mx::take(z, inv_order, 0);                              // _scatter_unsort(1162)
  if (!miss_rows.empty()) {
    std::map<int64_t, std::vector<int64_t>> by_expert;
    for (auto i : miss_rows) by_expert[flat[i]].push_back(i);
    auto &cx = exec_counters();
    cx.qmm_calls.fetch_add(3 * (uint64_t)by_expert.size(), std::memory_order_relaxed);
    cx.qmm_experts.fetch_add(3 * (uint64_t)by_expert.size(), std::memory_order_relaxed);
    cx.leases.fetch_add(by_expert.size(), std::memory_order_relaxed);
    for (auto &[e, tk] : by_expert) {
      auto b = io_->build(li, e);
      if (!b) { err = "hot miss build L" + std::to_string(li); return false; }
      std::vector<int32_t> rows, fpos;
      for (auto i : tk) { rows.push_back((int32_t)(i / k)); fpos.push_back((int32_t)i); }  // :1169,1217-1218
      const int64_t m = (int64_t)rows.size();
      auto rarr = mx::array(rows.begin(), mkshape({m}), mx::int32);
      auto xe_m = mx::reshape(mx::take(x_flat, rarr, 0), mkshape({m, 1, 1, H_}));  // :1176-1177
      auto one = mx::zeros(mkshape({1}), mx::int32);                                // :1182
      auto e1 = [&](int p) {  // _e1 view-like reshape(:1180-1181)
        return mx::reshape(b->at(p), mkshape({1, b->at(p).shape(0), b->at(p).shape(1)}));
      };
      auto gqm = [&](const mx::array &xx, int o) {
        return mx::gather_qmm(xx, e1(o), e1(o + 1), e1(o + 2), std::nullopt, one, true, 64, 4,
                              "affine", true);
      };
      auto gu = gqm(xe_m, 0);
      auto gg = gqm(xe_m, 3);
      auto zm = mx::gather_qmm((gg * mx::sigmoid(gg)) * gu, e1(6), e1(7), e1(8), std::nullopt,
                               one, true, 64, 4, "affine", true);                    // :1207-1215
      auto ze = mx::squeeze(mx::squeeze(zm, -2), -2);                                // :1216 [m,H]
      auto farr = mx::array(fpos.begin(), mkshape({m}), mx::int32);
      out = mx::scatter_add(
          out, {farr}, mx::reshape(ze, mkshape({m, 1, 1, H_})), {0});
    }
  }
  routed = mx::reshape(out, mkshape({1, T, k, H_}));
  auto &cx = exec_counters();
  cx.qmm_calls.fetch_add(3, std::memory_order_relaxed);
  cx.qmm_experts.fetch_add(3 * (uint64_t)std::min<int64_t>(n_rows, TK),
                           std::memory_order_relaxed);
  cx.qmm_weight_bytes.fetch_add(
      (uint64_t)std::min<int64_t>(n_rows, TK) *
          (2 * (F * (H_ / 8) * 4 + 2 * F * (H_ / 64) * 2) +
           (H_ * (F / 8) * 4 + 2 * H_ * (F / 64) * 2)),
      std::memory_order_relaxed);
  return true;
}

bool Net35::step(int32_t token, State35 &st, std::vector<float> &logits,
                 std::vector<RouteRec> &recs, std::string &err, DecodeProbe *probe) {
  try {
    step_bundles_.clear();
    std::vector<float> xh;
    if (!embed_lookup(embed_, {token}, H_, V_, xh, err)) return false;
    mx::array x = mx::astype(mx::reshape(wrap_f32(xh, mkshape({1, 1, H_})), mkshape({1, 1, H_})),
                             mx::bfloat16);
    recs.clear();
    for (size_t li = 0; li < layers_.size(); li++) {
      RouteRec rec;
      if (!layer_fwd(layers_[li], static_cast<int>(li), x, st, true, rec, nullptr, err))
        return false;
      recs.push_back(std::move(rec));
    }
    mx::array xs = e0_rms_norm(
        x, mx::astype(wrap_f32(final_norm_w_, mkshape({H_})), mx::bfloat16), eps_);
    if (!to_host32(lm_head_.apply(mx::reshape(xs, mkshape({1, H_}))), logits, err)) return false;
    st.position += 1;
    if (exec_mode() == ExecMode::Batched) {
      if (!stage_all_gpu(st, err, probe)) return false;
    } else if (probe) {
      probe->owners.clear(); probe->m_in.clear(); probe->logits.clear();
      for (int h = 0; h < pr_->n_heads(); h++) {
        int own = pr_->owner(h);
        if (!st.pr.cap_hidden[own].empty()) {
          probe->owners.push_back(own);
          probe->m_in.push_back(st.pr.cap_hidden[own]);
        }
      }
      std::vector<std::vector<double>> diag;
      if (!st.pr.run_step_boundary(*pr_, &diag, err)) return false;
      probe->logits = std::move(diag);
    } else if (!st.pr.run_step_boundary(*pr_, nullptr, err)) return false;
  } catch (const std::exception &ex) {
    err = std::string("step: ") + ex.what();
    return false;
  }
  return true;
}

}  // namespace e0n