Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
29.3 KB
Raw Blame
#include "expertio.hpp"

#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <set>
#include <vector>

#include "fwd_common.hpp"  // mkshape

namespace e0n {
namespace mx = mlx::core;
// ---------------- ShardMaps ----------------

ExpertIo::ShardMaps::~ShardMaps() {
  std::lock_guard<std::mutex> lk(mu_);
  for (auto &[p, m] : maps_) munmap(m.first, size_t(m.second));
  for (auto &[p, fd] : fds_) ::close(fd);
}

const uint8_t *ExpertIo::ShardMaps::raw(const std::string &shard_path, uint64_t off,
                                        uint64_t len) {
  uint8_t *base = nullptr;
  uint64_t total = 0;
  {
    std::lock_guard<std::mutex> lk(mu_);
    auto it = maps_.find(shard_path);
    if (it == maps_.end()) {
      int fd = ::open(shard_path.c_str(), O_RDONLY);
      if (fd < 0) return nullptr;
      struct stat sb;
      if (fstat(fd, &sb) != 0) { ::close(fd); return nullptr; }
      void *m = mmap(nullptr, size_t(sb.st_size), PROT_READ, MAP_SHARED, fd, 0);
      if (m == MAP_FAILED) { ::close(fd); return nullptr; }
      int keep = fd;
      it = maps_.emplace(shard_path,
                         std::pair<uint8_t *, uint64_t>{static_cast<uint8_t *>(m),
                                                        uint64_t(sb.st_size)}).first;
      fds_.emplace(shard_path, keep);
    }
    base = it->second.first;
    total = it->second.second;
  }
  if (off + len > total) return nullptr;
  return base + off;
}

void ExpertIo::ShardMaps::advise_range(const std::string &shard_path, uint64_t off,
                                       uint64_t len) {
  if (len == 0) return;
  if (!raw(shard_path, 0, 1)) return;
  std::lock_guard<std::mutex> lk(mu_);
  auto it = maps_.find(shard_path);
  if (it == maps_.end()) return;
  uint64_t page = uint64_t(sysconf(_SC_PAGESIZE));
  uint64_t start = off / page * page;
  uint64_t end = (off + len + page - 1) / page * page;
  if (end > it->second.second) end = it->second.second;
  if (end > start) madvise(it->second.first + start, size_t(end - start), MADV_WILLNEED);
}

void ExpertIo::ShardMaps::advise_whole(const std::string &shard_path) {  // mmap.py:43-49
  if (!raw(shard_path, 0, 1)) return;
  std::lock_guard<std::mutex> lk(mu_);
  auto it = maps_.find(shard_path);
  if (it != maps_.end()) madvise(it->second.first, size_t(it->second.second), MADV_WILLNEED);
}

void ExpertIo::ShardMaps::seq_read(const std::string &shard_path) {
  if (!raw(shard_path, 0, 1)) return;
  uint64_t total;
  {
    std::lock_guard<std::mutex> lk(mu_);
    total = maps_[shard_path].second;
  }
  volatile uint64_t sink = 0;
  constexpr uint64_t kChunk = 1ull << 24;
  for (uint64_t off = 0; off < total; off += kChunk) {
    uint64_t n = std::min(kChunk, total - off);
    const uint8_t *p = raw(shard_path, off, n);
    for (uint64_t o = 0; o + 8 <= n; o += 8) {
      uint64_t v;
      std::memcpy(&v, p + o, 8);
      sink += v;
    }
  }
  (void)sink;
}

std::shared_ptr<Bundle> SharedExpertCache::get(uint64_t key) {  // cache.py:24-29
  std::lock_guard<std::mutex> lk(mu_);
  auto it = map_.find(key);
  if (it == map_.end()) return nullptr;
  for (auto o = order_.begin(); o != order_.end(); ++o)
    if (*o == key) { order_.erase(o); order_.push_back(key); break; }
  return it->second;
}
std::shared_ptr<Bundle> SharedExpertCache::peek(uint64_t key) {  // 31-33
  std::lock_guard<std::mutex> lk(mu_);
  auto it = map_.find(key);
  return it == map_.end() ? nullptr : it->second;
}
void SharedExpertCache::put(uint64_t key, std::shared_ptr<Bundle> b) {  // 34-39
  std::lock_guard<std::mutex> lk(mu_);
  if (map_.count(key)) {
    map_[key] = std::move(b);
    for (auto o = order_.begin(); o != order_.end(); ++o)
      if (*o == key) { order_.erase(o); order_.push_back(key); break; }
    return;
  }
  map_[key] = std::move(b);
  order_.push_back(key);
  if (slots_ > 0 && int(map_.size()) > slots_) {  // popitem(last=False)
    uint64_t old = order_.front();
    order_.pop_front();
    map_.erase(old);
    evictions_++;
  }
}
std::shared_ptr<Bundle> SharedExpertCache::pop(uint64_t key) {  // 41-46
  std::lock_guard<std::mutex> lk(mu_);
  auto it = map_.find(key);
  if (it == map_.end()) return nullptr;
  auto b = it->second;
  map_.erase(it);
  for (auto o = order_.begin(); o != order_.end(); ++o)
    if (*o == key) { order_.erase(o); break; }
  return b;
}
size_t SharedExpertCache::size() {
  std::lock_guard<std::mutex> lk(mu_);
  return map_.size();
}
bool SharedExpertCache::contains(uint64_t key) {
  std::lock_guard<std::mutex> lk(mu_);
  return map_.count(key) > 0;
}

bool PrefetchBuffer::put(uint64_t key, std::shared_ptr<Bundle> b) {  // cache.py:66-72
  std::lock_guard<std::mutex> lk(mu_);
  if (map_.count(key)) { map_[key] = std::move(b); return true; }
  map_[key] = std::move(b);
  order_.push_back(key);
  if (cap_ > 0 && int(map_.size()) > cap_) {
    uint64_t old = order_.front();
    order_.pop_front();
    map_.erase(old);
    return false;
  }
  return true;
}
std::shared_ptr<Bundle> PrefetchBuffer::pop(uint64_t key) {  // 74-75
  std::lock_guard<std::mutex> lk(mu_);
  auto it = map_.find(key);
  if (it == map_.end()) return nullptr;
  auto b = it->second;
  map_.erase(it);
  for (auto o = order_.begin(); o != order_.end(); ++o)
    if (*o == key) { order_.erase(o); break; }
  return b;
}
bool PrefetchBuffer::contains(uint64_t key) {
  std::lock_guard<std::mutex> lk(mu_);
  return map_.count(key) > 0;
}

// ---------------- IoPool ----------------

IoPool::IoPool(int n) {
  for (int i = 0; i < n; i++)
    th_.emplace_back([this] {
      for (;;) {
        std::function<void()> job;
        {
          std::unique_lock<std::mutex> lk(mu_);
          cv_.wait(lk, [this] { return stop_ || !q_.empty(); });
          if (stop_ && q_.empty()) return;
          job = std::move(q_.front());
          q_.pop_front();
        }
        job();
      }
    });
}
IoPool::~IoPool() {
  {
    std::lock_guard<std::mutex> lk(mu_);
    stop_ = true;
  }
  cv_.notify_all();
  for (auto &t : th_)
    if (t.joinable()) t.join();
}

// ---------------- ExpertIo ----------------

ExpertIo::ExpertIo(const ModelFiles *files, int cache_slots, int prefetch_cap, int load_threads,
                   int prefetch_threads)
    : files_(files), cache_(cache_slots), pbuf_(prefetch_cap), load_pool_(load_threads),
      prefetch_pool_(prefetch_threads) {}

namespace {
void materialize(const std::shared_ptr<Bundle> &b) {
  std::vector<mx::array> v;
  for (int i = 0; i < 9; i++) v.push_back(b->at(i));
  mx::eval(v);
}

const char *kProjFile[3] = {"up_proj", "gate_proj", "down_proj"};
const char *kPartFile[3] = {"weight", "scales", "biases"};
}  // namespace

void ExpertIo::add_layer(const LayerCfg &cfg) {
  li2idx_[cfg.layer] = int(layers_.size());
  layers_.emplace_back();
  Layer &L = layers_.back();
  L.cfg = cfg;
  for (int p = 0; p < 3; p++)
    for (int q = 0; q < 3; q++) {
      const int i = p * 3 + q;
      const std::string name = cfg.prefix + "." + kProjFile[p] + "." + kPartFile[q];
      const TensorInfo *ti = files_->find(name);
      if (!ti) {
        std::fprintf(stderr, "expertio: missing tensor %s\n", name.c_str());
        continue;
      }
      const int64_t O = (p == 2) ? cfg.H : cfg.F;
      const int64_t K = (q == 0) ? ((p == 2) ? cfg.F / 8 : cfg.H / 8)
                                 : ((p == 2) ? cfg.F / cfg.group : cfg.H / cfg.group);
      L.reg[i] = {ti, O, K, (ti->end - ti->begin) / uint64_t(cfg.E), q == 0 ? 4u : 2u};
      if (ti->dtype == "BF16") L.reg[i].esize = 2;
      else if (ti->dtype == "U32") L.reg[i].esize = 4;
      else std::fprintf(stderr, "expertio: unexpected dtype %s @%s\n", ti->dtype.c_str(), name.c_str());
    }
}

int ExpertIo::layer_idx(int li) const {
  auto it = li2idx_.find(li);
  return it == li2idx_.end() ? -1 : it->second;
}

std::shared_ptr<Bundle> ExpertIo::build(int li, int64_t expert) {
  int idx = layer_idx(li);
  if (idx < 0) return nullptr;
  Layer &L = layers_[size_t(idx)];
  auto b = std::make_shared<Bundle>();
  uint64_t tot = 0;
  for (int i = 0; i < 9; i++) {
      const auto &r = L.reg[i];
      if (!r.ti) return nullptr;
      const uint8_t *src = shards_.raw(files_->shard_path(r.ti->shard),
                                       r.ti->begin + uint64_t(expert) * r.per, r.per);
      if (!src) {
        std::fprintf(stderr, "expertio: raw out of range %s e%lld off=%llu per=%llu\n",
                     files_->shard_path(r.ti->shard).c_str(), (long long)expert,
                     (unsigned long long)(r.ti->begin + uint64_t(expert) * r.per),
                     (unsigned long long)r.per);
        return nullptr;
      }
      if (!r.ti->shape.empty() && r.per != uint64_t(r.rows) * uint64_t(r.cols) * r.esize) {
        std::fprintf(stderr, "expertio: region size mismatch %s per=%llu vs %lldx%lldx%u\n",
                     r.ti->dtype.c_str(), (unsigned long long)r.per, (long long)r.rows,
                     (long long)r.cols, r.esize);
        return nullptr;
      }
      if (r.esize == 4) {
        b->at_mut(i) = mx::array(reinterpret_cast<const uint32_t *>(src),
                                 mkshape({r.rows, r.cols}), mx::uint32);
      } else {
        auto a16 = mx::array(reinterpret_cast<const uint16_t *>(src), mkshape({r.rows, r.cols}),
                             mx::uint16);
        b->at_mut(i) = mx::view(a16, mx::bfloat16);
      }
      tot += r.per;
  }
  b->bytes = tot;
  heap_->add(tot);
  auto heap = heap_;
  auto keep = b;
  return std::shared_ptr<Bundle>(b.get(), [heap, tot, keep](Bundle *p) mutable {
    heap->sub(tot);
    keep.reset();
  });
}

std::map<int64_t, std::shared_ptr<Bundle>> ExpertIo::get_bundles(
    int li, const std::vector<int64_t> &experts) {
  std::map<int64_t, std::shared_ptr<Bundle>> out;
  std::vector<int64_t> missing;
  int lidx = layer_idx(li);
  Layer *L = (lidx >= 0) ? &layers_[size_t(lidx)] : nullptr;
  const double decay = L ? L->cfg.hot_decay : 0.75;

  for (auto e : experts) {  // 430-435
    auto c = cache_.get(key_of(li, e));
    if (c) { out[e] = std::move(c); hits_++; }
    else missing.push_back(e);
  }

  std::vector<int64_t> still;  // 437-449
  for (auto e : missing) {
    auto pb = pbuf_.pop(key_of(li, e));
    if (pb) {
      phi_++; hits_++;
      if (L) L->bump_count(e, decay);
      cache_.put(key_of(li, e), pb);
      out[e] = std::move(pb);
    } else still.push_back(e);
  }

  std::vector<int64_t> cold;  // 451-467 inflight
  for (auto e : still) {
    std::shared_future<std::shared_ptr<Bundle>> fut;
    {
      std::lock_guard<std::mutex> lk(infl_mu_);
      auto it = inflight_.find(key_of(li, e));
      if (it != inflight_.end()) fut = it->second;
    }
    if (fut.valid()) {
        auto b = fut.get();
        {
        std::lock_guard<std::mutex> lk(infl_mu_);
        inflight_.erase(key_of(li, e));
      }
      if (!b) {
        std::fprintf(stderr, "expertio: inflight build failed L%d E%lld\n", li, (long long)e);
        continue;
      }
      phi_++; hits_++;
      if (L) L->bump_count(e, decay);
      cache_.put(key_of(li, e), b);
      out[e] = std::move(b);
    } else cold.push_back(e);
  }
  if (!cold.empty()) {
    std::vector<std::shared_future<std::shared_ptr<Bundle>>> fs;
    for (auto e : cold) fs.push_back(load_pool_.submit([this, li, e] { return build(li, e); }));
    for (size_t i = 0; i < cold.size(); i++) {
      auto b = fs[i].get();
      if (!b) {
        std::fprintf(stderr, "expertio: build failed L%d E%lld (missing region key or map OOB)\n", li,
                     (long long)cold[i]);
        continue;
      }
      loads_++; misses_++;
      if (L) L->bump_count(cold[i], decay);
      cache_.put(key_of(li, cold[i]), b);
      out[cold[i]] = std::move(b);
    }
  }
  for (auto &[e, b] : out) materialize(b);
  return out;
}

void ExpertIo::prefetch(int li, const std::vector<int64_t> &experts) {
  std::vector<int64_t> uniq(experts.begin(), experts.end());
  std::sort(uniq.begin(), uniq.end());
  uniq.erase(std::unique(uniq.begin(), uniq.end()), uniq.end());
  if (uniq.empty()) return;
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  std::vector<int64_t> missing;
  for (auto e : uniq) {
    {
      std::lock_guard<std::mutex> lk(infl_mu_);
      if (inflight_.count(key_of(li, e))) continue;
    }
    if (cache_.peek(key_of(li, e)) || pbuf_.contains(key_of(li, e))) continue;
    missing.push_back(e);
  }
  if (missing.empty()) return;
  if (L.cfg.warm_willneed) warm_willneed(li, missing);  // 502-503
  std::vector<std::pair<int64_t, std::shared_future<std::shared_ptr<Bundle>>>> sub;
  for (auto e : missing) {
    auto fut = prefetch_pool_.submit([this, li, e] { return build(li, e); });
    {
      std::lock_guard<std::mutex> lk(infl_mu_);
      inflight_[key_of(li, e)] = fut;
    }
    sub.emplace_back(e, fut);
  }
  psub_ += missing.size();
  for (auto &[e, fut] : sub) {
    uint64_t key = key_of(li, e);
    prefetch_pool_.submit([this, e, key, fut] {
      auto b = fut.get();
      {
        std::lock_guard<std::mutex> lk(infl_mu_);
        auto it = inflight_.find(key);
        if (it != inflight_.end()) inflight_.erase(it);  // 530
      }
      if (cache_.peek(key)) return;                       // 531
      if (!pbuf_.put(key, b)) pwaste_++;                  // 533-535
    });
  }
}

void ExpertIo::warm_willneed(int li, const std::vector<int64_t> &experts) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  for (auto e : experts)
    for (int i = 0; i < 9; i++) {
      if (!L.reg[i].ti) continue;
      shards_.advise_range(files_->shard_path(L.reg[i].ti->shard),
                           L.reg[i].ti->begin + uint64_t(e) * L.reg[i].per, L.reg[i].per);
    }
}

void ExpertIo::warm_pages(int li, const std::vector<int64_t> &experts) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  volatile uint8_t sink = 0;
  uint64_t page = uint64_t(sysconf(_SC_PAGESIZE));
  for (auto e : experts)
    for (int i = 0; i < 9; i++) {
      if (!L.reg[i].ti) continue;
      const uint8_t *src = shards_.raw(files_->shard_path(L.reg[i].ti->shard),
                                       L.reg[i].ti->begin + uint64_t(e) * L.reg[i].per,
                                       L.reg[i].per);
      if (!src) continue;
      for (uint64_t o = 0; o < L.reg[i].per; o += page) sink ^= src[o];
    }
  (void)sink;
}

void ExpertIo::shard_advise_whole(const std::string &shard_name) {
  shards_.advise_whole(files_->shard_path(shard_name));
}
void ExpertIo::shard_seq_read(const std::string &shard_name) {
  shards_.seq_read(files_->shard_path(shard_name));
}

void ExpertIo::stage_experts(int li, const std::vector<int64_t> &experts) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  std::vector<int64_t> uniq(experts.begin(), experts.end());
  std::sort(uniq.begin(), uniq.end());
  uniq.erase(std::unique(uniq.begin(), uniq.end()), uniq.end());
  if (uniq.empty()) return;  // 721-722
  if (L.cfg.warm_willneed) warm_willneed(li, uniq);  // 723-724
  std::lock_guard<std::mutex> lk(*L.mu);
  if (L.cfg.incr_mode) {
    stage_incr_locked(L, uniq);
    return;
  }
  // 733-738:next = _build_staged_bundles(unique)(539-585).
  const int n = L.cfg.staged_n;
  StagedState st;
  st.exp.assign(uniq.begin(), uniq.begin() + std::min<int>(size_t(n), uniq.size()));
  st.active = true;
  for (auto e : st.exp) {
    std::shared_ptr<Bundle> b;
    for (size_t i = 0; i < L.state.exp.size(); i++)
      if (L.state.exp[i] == e) { b = L.state.bundles[i]; break; }
    if (!b) b = cache_.get(key_of(li, e));
    if (!b) {
      b = build(li, e);
      if (!b) return;
      cache_.put(key_of(li, e), b);
      sbuilds_++;
    }
    st.bundles.push_back(std::move(b));
  }
  L.next = std::move(st);
}

void ExpertIo::wait_staged(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  std::lock_guard<std::mutex> lk(*L.mu);
  if (L.next.active) {
    L.state = L.next;
    L.next = StagedState{};
  }
}
const ExpertIo::StagedState &ExpertIo::staged(int li) const {
  return layers_[size_t(layer_idx(li))].state;
}
const ExpertIo::StagedState &ExpertIo::next_staged(int li) const {
  return layers_[size_t(layer_idx(li))].next;
}

void ExpertIo::stage_incr_locked(Layer &L, const std::vector<int64_t> &uniq) {
  const int li = L.cfg.layer;
  const int n = L.cfg.staged_n;
  std::vector<int64_t> exp(uniq.begin(), uniq.begin() + std::min<int>(size_t(n), uniq.size()));
  std::set<int64_t> exp_set(exp.begin(), exp.end());
  if (L.incr_tensors.empty()) {
    for (int i = 0; i < 9; i++) {
      auto t = mx::zeros(mkshape({int64_t(n) + 1, L.reg[i].rows, L.reg[i].cols}),
                         L.reg[i].esize == 4 ? mx::uint32 : mx::bfloat16);
      mx::eval(t);
      L.incr_tensors.push_back(std::move(t));
    }
    L.incr_table = mx::full(mkshape({L.cfg.E}), int32_t(n), mx::int32);  // core.full(614-616)
    mx::eval(L.incr_table);
    L.occ.assign(n, -1);
  }
  auto &occ = L.occ;
  for (int i = 0; i < n; i++)
    if (occ[i] >= 0 && !exp_set.count(occ[i])) occ[i] = -1;
  std::vector<int> free;
  for (int i = 0; i < n; i++) if (occ[i] < 0) free.push_back(i);
  std::vector<int64_t> changed;  // 633-637
  for (auto e : exp) if (!L.incr_bundles.count(e)) changed.push_back(e);
  std::map<int64_t, std::shared_ptr<Bundle>> bundles;
  for (auto e : exp) if (L.incr_bundles.count(e)) bundles[e] = L.incr_bundles[e];
  struct Write { int slot; std::shared_ptr<Bundle> b; };
  std::vector<Write> writes;
  for (size_t j = 0; j < changed.size(); j++) {  // 646-668
    int64_t e = changed[j];
    if (int(j) >= int(free.size())) continue;
    std::shared_ptr<Bundle> b = cache_.get(key_of(li, e));
    if (!b) {
      b = build(li, e);
      if (!L.cfg.incr_writeback && b) cache_.put(key_of(li, e), b);
      sbuilds_++;
    }
    int slot = free[j];
    materialize(b);
    bundles[e] = b;
    occ[slot] = e;
    writes.push_back({slot, std::move(b)});
  }
  std::vector<int64_t> evicted;  // 669-670
  for (auto &[e, b] : L.incr_bundles)
    if (!exp_set.count(e)) evicted.push_back(e);
  if (L.cfg.incr_writeback)
    for (auto e : evicted) cache_.put(key_of(li, e), L.incr_bundles[e]);
  L.incr_bundles = std::move(bundles);
  for (auto &w : writes)
    for (int i = 0; i < 9; i++) {
      uint64_t row = uint64_t(L.reg[i].rows) * uint64_t(L.reg[i].cols);
      if (L.reg[i].esize == 4)
        std::memcpy(L.incr_tensors[i].data<uint32_t>() + uint64_t(w.slot) * row,
                    w.b->at(i).data<uint32_t>(), row * 4);
      else
        std::memcpy(L.incr_tensors[i].data<uint16_t>() + uint64_t(w.slot) * row,
                    w.b->at(i).data<uint16_t>(), row * 2);
    }
  int32_t *tbl = L.incr_table.data<int32_t>();
  for (auto e : evicted) tbl[e] = int32_t(n);
  for (int i = 0; i < n; i++)
    if (occ[i] >= 0) tbl[occ[i]] = int32_t(i);
  if (!writes.empty() || !evicted.empty() || !changed.empty()) mx::eval(L.incr_table);
  StagedState st;
  st.active = true;
  st.exp = exp;
  for (auto e : exp) st.bundles.push_back(L.incr_bundles[e]);
  L.next = std::move(st);
}

bool ExpertIo::incr_mode(int li) const {
  int idx = layer_idx(li);
  return idx >= 0 && layers_[size_t(idx)].cfg.incr_mode;
}
bool ExpertIo::incr_ready(int li) const {
  int idx = layer_idx(li);
  return idx >= 0 && layers_[size_t(idx)].cfg.incr_mode &&
         !layers_[size_t(idx)].incr_tensors.empty();
}
const std::vector<mx::array> &ExpertIo::incr_tensors(int li) {
  return layers_[size_t(layer_idx(li))].incr_tensors;
}
const mx::array &ExpertIo::incr_slot_table(int li) const {
  return layers_[size_t(layer_idx(li))].incr_table;
}

static std::string exp_key(const std::vector<int64_t> &exp) {
  std::string s;
  for (auto e : exp) { s += std::to_string(e); s += ','; }
  return s;
}

ExpertIo::Asm ExpertIo::asm_for(int li, const StagedState &st) {
  Layer &L = layers_[size_t(layer_idx(li))];
  const std::string key = exp_key(st.exp);
  if (L.cfg.asm_cache) {
    auto it = L.asm_cache.find(key);
    if (it != L.asm_cache.end() && it->second.exp == st.exp) return it->second;
  }
  const int n = L.cfg.staged_n;
  if (L.zero_rows.empty()) {
    for (int i = 0; i < 9; i++)
      L.zero_rows.push_back(mx::zeros(mkshape({L.reg[i].rows, L.reg[i].cols}),
                                      L.reg[i].esize == 4 ? mx::uint32 : mx::bfloat16));
  }
  Asm a;
  a.exp = st.exp;
  auto sit = L.slot_cache.find(key);  // _slot_table_cache(1269-1276)
  if (sit == L.slot_cache.end()) {
    std::vector<int32_t> tab(size_t(L.cfg.E), int32_t(n));  // 557-559
    for (size_t i = 0; i < st.exp.size(); i++) tab[size_t(st.exp[i])] = int32_t(i);
    mx::array tof(tab.begin(), mkshape({L.cfg.E}), mx::int32);  // core.array(list)
    mx::eval(tof);  // 1271-1272 force eval
    L.slot_cache.insert_or_assign(key, tof);
    L.slot_keys.push_back(key);
    if (L.slot_keys.size() > 8) {
      L.slot_cache.erase(L.slot_keys.front());
      L.slot_keys.pop_front();
    }
    a.slot_of = L.slot_cache.at(key);
  } else {
    a.slot_of = sit->second;
  }
  for (int i = 0; i < 9; i++) {
    std::vector<mx::array> rows;
    for (auto &b : st.bundles) rows.push_back(b->at(i));
    while (int(rows.size()) < n + 1) rows.push_back(L.zero_rows[i]);
    a.wargs.push_back(mx::stack(rows));
  }
  sused_ += st.exp.size();
  if (L.cfg.asm_cache) {
    if (!L.asm_cache.empty()) L.asm_cache.clear();  // cap=1
    L.asm_cache.insert_or_assign(key, a);
  }
  return a;
}

void ExpertIo::note_call(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  L.calls++;
  if (L.cfg.hot_update_interval && L.calls % L.cfg.hot_update_interval == 0) {
    (void)L.cfg.hot_per_layer;
  }
}

void ExpertIo::refresh_hot_pins(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  if (layers_[size_t(idx)].cfg.hot_per_layer <= 0) return;
}

void ExpertIo::load_hot_layer(int li, int n_hot) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  std::vector<int64_t> hot;
  if (!L.hc_order.empty()) {
    hot = L.hc_order;
    std::stable_sort(hot.begin(), hot.end(),
                     [&](int64_t a, int64_t b) { return L.hc_val[a] > L.hc_val[b]; });
    if ((int64_t)hot.size() > n_hot) hot.resize(n_hot);  // hot[:n_hot](1007)
  } else {
    for (int64_t e = 0; e < n_hot; e++) hot.push_back(e);  // 1009
  }
  if (L.hot_backing) {
    if (L.hot_key == hot) return;  // 1012-1013
    std::set<int64_t> old_set(L.hot_key.begin(), L.hot_key.end());
    int changed = 0;
    for (auto e : hot)
      if (!old_set.count(e)) changed++;
    if (changed <= std::max(1, n_hot / 8)) return;  // 1016-1017
  }
  auto st = std::make_shared<Layer::HotStack>();
  st->key = hot;
  st->sorted = st->key;
  std::sort(st->sorted.begin(), st->sorted.end());
  {
    std::lock_guard<std::mutex> lk(*L.backing_mu);
    L.hot_backing = st;
  }
  L.hot_key = st->key;
  if (!L.hot_weights.empty()) {
    heap_->sub(L.hot_heap);
    L.hot_heap = 0;
    L.hot_weights.clear();
  }
}

void ExpertIo::materialize_hot(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  if (!L.hot_weights.empty() || !L.hot_backing) return;  // 1053
  std::shared_ptr<const Layer::HotStack> snap;
  {
    std::lock_guard<std::mutex> lk(*L.backing_mu);
    snap = L.hot_backing;
  }
  const int64_t n_e = (int64_t)snap->key.size() + 1;
  L.hot_weights.assign(9, mx::array(0.f));
  for (int i = 0; i < 9; i++) {
    const auto &r = L.reg[i];
    if (!r.ti) return;
    std::vector<uint8_t> rows(static_cast<size_t>(n_e) * r.per, 0);
    const std::string path = files_->shard_path(r.ti->shard);
    for (size_t j = 0; j < snap->key.size(); j++) {
      const uint8_t *src = shards_.raw(path, r.ti->begin + uint64_t(snap->key[j]) * r.per,
                                       r.per);
      if (!src) return;
      std::memcpy(rows.data() + uint64_t(j) * r.per, src, r.per);
    }
    const uint8_t *base = rows.data();
    if (r.esize == 4) {
      L.hot_weights[i] = mx::array(reinterpret_cast<const uint32_t *>(base),
                                   mkshape({n_e, r.rows, r.cols}), mx::uint32);
    } else {
      auto a16 = mx::array(reinterpret_cast<const uint16_t *>(base),
                           mkshape({n_e, r.rows, r.cols}), mx::uint16);
      L.hot_weights[i] = mx::view(a16, mx::bfloat16);
    }
    L.hot_heap += rows.size();
  }
  heap_->add(L.hot_heap);
}

void ExpertIo::dematerialize_hot(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  if (L.hot_weights.empty()) return;
  heap_->sub(L.hot_heap);
  L.hot_heap = 0;
  L.hot_weights.clear();
}

void ExpertIo::clear_hot_layer(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  if (!L.hot_weights.empty()) {
    heap_->sub(L.hot_heap);
    L.hot_heap = 0;
    L.hot_weights.clear();
  }
  std::lock_guard<std::mutex> lk(*L.backing_mu);
  L.hot_backing = nullptr;
  L.hot_key.clear();
}

void ExpertIo::load_full_layer(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  if (!L.full_w.empty()) return;  // 934-935
  const int64_t E = L.cfg.E;
  L.full_w.assign(9, mx::array(0.f));
  uint64_t tot = 0;
  for (int i = 0; i < 9; i++) {
    const auto &r = L.reg[i];
    if (!r.ti) {
      std::fprintf(stderr, "expertio: load_full_layer missing region L%d r%d\n", li, i);
      return;
    }
    const uint64_t span = uint64_t(E) * r.per;
    const uint8_t *raw = shards_.raw(files_->shard_path(r.ti->shard), r.ti->begin, span);
    if (!raw) {
      std::fprintf(stderr, "expertio: load_full_layer raw OOB L%d span=%llu\n", li,
                   (unsigned long long)span);
      return;
    }
    if (r.esize == 4) {  // 974-975 u32_view(raw,shape) → core.array
      L.full_w[i] =
          mx::array(reinterpret_cast<const uint32_t *>(raw), mkshape({E, r.rows, r.cols}),
                    mx::uint32);
    } else {  // 976-978 raw.view(<u2) → core.array → .view(bf16)
      auto a16 = mx::array(reinterpret_cast<const uint16_t *>(raw),
                           mkshape({E, r.rows, r.cols}), mx::uint16);
      L.full_w[i] = mx::view(a16, mx::bfloat16);
    }
    tot += span;
  }
  L.full_heap = tot;
  heap_->add(tot);
}

void ExpertIo::clear_full_layer(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  if (L.full_w.empty()) return;
  heap_->sub(L.full_heap);
  L.full_heap = 0;
  L.full_w.clear();
}

bool ExpertIo::hot_ready(int li) const {
  int idx = layer_idx(li);
  return idx >= 0 && !layers_[size_t(idx)].hot_weights.empty();  // 1121 `_hot_weights is not None`
}
bool ExpertIo::full_ready(int li) const {
  int idx = layer_idx(li);
  return idx >= 0 && !layers_[size_t(idx)].full_w.empty();  // 1231 `_full_weights is not None`
}
const std::vector<mx::array> &ExpertIo::hot_weights(int li) {
  static const std::vector<mx::array> kEmpty;
  int idx = layer_idx(li);
  return idx >= 0 ? layers_[size_t(idx)].hot_weights : kEmpty;
}
const std::vector<mx::array> &ExpertIo::full_weights(int li) {
  static const std::vector<mx::array> kEmpty;
  int idx = layer_idx(li);
  return idx >= 0 ? layers_[size_t(idx)].full_w : kEmpty;
}
const std::vector<int64_t> &ExpertIo::hot_key(int li) const {
  static const std::vector<int64_t> kEmpty;
  int idx = layer_idx(li);
  return idx >= 0 ? layers_[size_t(idx)].hot_key : kEmpty;
}
const std::vector<int64_t> &ExpertIo::last_used(int li) const {
  static const std::vector<int64_t> kEmpty;
  int idx = layer_idx(li);
  return idx >= 0 ? layers_[size_t(idx)].last_used : kEmpty;
}

void ExpertIo::note_prefill_use(int li, const std::vector<int64_t> &flat, int k,
                                bool staged_mode) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  std::vector<int64_t> uniq = flat;
  std::sort(uniq.begin(), uniq.end());
  uniq.erase(std::unique(uniq.begin(), uniq.end()), uniq.end());
  for (auto e : uniq) L.bump_count(e, L.cfg.hot_decay);  // 1125-1127
  if (staged_mode && (int)flat.size() >= k)
    L.last_prefill_topk.assign(flat.end() - k, flat.end());  // 1128-1130
}

void ExpertIo::prefetch_from_prefill(int li) {
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  if (L.last_prefill_topk.empty()) return;  // 887-888
  std::vector<int64_t> uniq = L.last_prefill_topk;
  std::sort(uniq.begin(), uniq.end());
  uniq.erase(std::unique(uniq.begin(), uniq.end()), uniq.end());
  if (uniq.empty()) return;  // 890
  L.last_used = uniq;        // 891
  prefetch(li, uniq);        // 892
}

// Cross-request memory plus incremental slot table cache.
void ExpertIo::reset_layer(int li) {  // layer.py:1429-1441
  int idx = layer_idx(li);
  if (idx < 0) return;
  Layer &L = layers_[size_t(idx)];
  std::lock_guard<std::mutex> lk(*L.mu);
  L.state = StagedState{};   // _staged_state = None(1436)
  L.next = StagedState{};    // _staged_state_next = None(1437)
  L.last_prefill_topk.clear();  // 1435
  L.last_used.clear();          // 1440
  if (!L.full_w.empty()) {
    heap_->sub(L.full_heap);
    L.full_heap = 0;
    L.full_w.clear();
  }
}

ExpertIo::Metrics ExpertIo::snapshot() const {
  Metrics m;
  m.loads = loads_.load();
  m.hits = hits_.load();
  m.misses = misses_.load();
  m.evictions = cache_.evictions();
  m.prefetch_submitted = psub_.load();
  m.prefetch_hits = phi_.load();
  m.prefetch_wasted = pwaste_.load();
  m.staged_used = sused_.load();
  m.staged_fallback = sfallback_.load();
  m.stage_builds = sbuilds_.load();
  m.heap_bytes = heap_->live.load();
  m.heap_peak = heap_->peak.load();
  m.cache_size = const_cast<SharedExpertCache &>(cache_).size();
  m.inflight = inflight_size();
  return m;
}

}  // namespace e0n