Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
9.5 KB
Raw Blame
// Expert load/staging: mmap shards, global LRU, prefetch buffer, and staged double-buffer.
// Stack/lazy-graph construction stays on the main thread; pool threads only produce Bundles.
// Prefetch done-callbacks are registered outside the lock (deadlock otherwise).
#pragma once
#include <atomic>
#include <condition_variable>
#include <cstdint>
#include <deque>
#include <functional>
#include <future>
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>

#include <mlx/mlx.h>

#include "safio.hpp"

namespace e0n {
namespace mx = mlx::core;

// 9-region bundle. Field order is the moe_math signature: (up,gate,down)×(w,s,b),
// not file key order (gate_proj before up_proj). Shape is the stacked tensor
// without the expert axis [O,K]; elements live in mlx-owned buffers.
struct Bundle {
  Bundle()
      : f{mx::array(0.f), mx::array(0.f), mx::array(0.f), mx::array(0.f), mx::array(0.f),
         mx::array(0.f), mx::array(0.f), mx::array(0.f), mx::array(0.f)} {}
  mx::array f[9];  // 0..2=up w/s/b, 3..5=gate w/s/b, 6..8=down w/s/b
  uint64_t bytes = 0;
  const mx::array &at(int p) const { return f[p]; }
  mx::array &at_mut(int p) { return f[p]; }
};

class SharedExpertCache {
 public:
  explicit SharedExpertCache(int slots) : slots_(slots) {}
  std::shared_ptr<Bundle> get(uint64_t key);
  std::shared_ptr<Bundle> peek(uint64_t key);
  void put(uint64_t key, std::shared_ptr<Bundle> b);
  std::shared_ptr<Bundle> pop(uint64_t key);
  bool contains(uint64_t key);
  size_t size();
  uint64_t evictions() const { return evictions_; }

 private:
  int slots_;
  uint64_t evictions_ = 0;
  std::mutex mu_;
  std::deque<uint64_t> order_;  // front = LRU
  std::map<uint64_t, std::shared_ptr<Bundle>> map_;
};

class PrefetchBuffer {
 public:
  explicit PrefetchBuffer(int cap) : cap_(cap) {}
  bool put(uint64_t key, std::shared_ptr<Bundle> b);     // true = kept, false = overflow drop
  std::shared_ptr<Bundle> pop(uint64_t key);
  bool contains(uint64_t key);
  void set_cap(int c) { cap_ = c; }

 private:
  int cap_;
  std::mutex mu_;
  std::deque<uint64_t> order_;
  std::map<uint64_t, std::shared_ptr<Bundle>> map_;
};

class IoPool {
 public:
  explicit IoPool(int n);
  ~IoPool();
  template <class F>
  auto submit(F &&f) -> std::shared_future<decltype(f())> {
    using R = decltype(f());
    auto pk = std::make_shared<std::packaged_task<R()>>(std::forward<F>(f));
    auto fut = pk->get_future().share();
    {
      std::lock_guard<std::mutex> lk(mu_);
      q_.emplace_back([pk] { (*pk)(); });
    }
    cv_.notify_one();
    return fut;
  }

 private:
  std::mutex mu_;
  std::condition_variable cv_;
  std::deque<std::function<void()>> q_;
  bool stop_ = false;
  std::vector<std::thread> th_;
};

class ExpertIo {
 public:
  struct LayerCfg {
    int layer = 0;
    std::string prefix;   // "model.layers.N.mlp.experts" / "...mlp.switch_mlp"
    int64_t E = 0;
    int64_t H = 0, F = 0;
    int group = 64;
    int staged_n = 8;            // prod_k8=8 / staged_k4=4
    bool incr_mode = false;      // 8b prod_k8 false; 35b staged_k4 true
    bool incr_writeback = false; // 35b true
    bool warm_willneed = false;  // 35b true / 8b false
    bool asm_cache = true;
    int prefill_hot = 0;             // 35b staged_k4=32 / 8b prod_k8=0
    bool full_layer_prefill = false; // 35b false / 8b true
    int prefill_full_layers = 0;
    int hot_per_layer = 0;
    int hot_update_interval = 4;
    double hot_decay = 0.75;
  };

  ExpertIo(const ModelFiles *files, int cache_slots, int prefetch_cap, int load_threads,
           int prefetch_threads);
  void add_layer(const LayerCfg &cfg);

  std::shared_ptr<Bundle> build(int li, int64_t expert);
  // LRU → prefetch_buf.pop (promote) → wait inflight (promote) → pool sync + LRU.
  std::map<int64_t, std::shared_ptr<Bundle>> get_bundles(int li,
                                                         const std::vector<int64_t> &experts);
  void prefetch(int li, const std::vector<int64_t> &experts);
  void warm_pages(int li, const std::vector<int64_t> &experts);
  void warm_willneed(int li, const std::vector<int64_t> &experts);
  void shard_advise_whole(const std::string &shard_name);
  void shard_seq_read(const std::string &shard_name);

  struct StagedState {
    std::vector<std::shared_ptr<Bundle>> bundles;
    std::vector<int64_t> exp;  // sorted(set)[:staged_n]
    bool active = false;
  };
  void stage_experts(int li, const std::vector<int64_t> &experts);  // main thread
  void wait_staged(int li);
  const StagedState &staged(int li) const;
  const StagedState &next_staged(int li) const;

  bool incr_mode(int li) const;
  bool incr_ready(int li) const;
  const std::vector<mx::array> &incr_tensors(int li);
  const mx::array &incr_slot_table(int li) const;

  // Non-incr consume path: slot table (set-key cache, cap 8) + 9×[n+1,...] stack (cap 1).
  // Returns a cheap copy (mx::array is a node pointer). asm_cache=false rebuilds each call.
  struct Asm {
    std::vector<mx::array> wargs;  // 9×[n+1,...]
    mx::array slot_of{0.f};        // [E] int32, eval'd GPU-resident table
    std::vector<int64_t> exp;
  };
  Asm asm_for(int li, const StagedState &st);

  void load_hot_layer(int li, int n_hot);
  void materialize_hot(int li);
  void dematerialize_hot(int li);
  void clear_hot_layer(int li);
  void load_full_layer(int li);
  void clear_full_layer(int li);
  bool hot_ready(int li) const;
  bool full_ready(int li) const;
  const std::vector<mx::array> &hot_weights(int li);
  const std::vector<mx::array> &full_weights(int li);
  const std::vector<int64_t> &hot_key(int li) const;
  void note_call(int li);
  void refresh_hot_pins(int li);
  void note_prefill_use(int li, const std::vector<int64_t> &flat, int k, bool staged_mode);
  void prefetch_from_prefill(int li);
  const std::vector<int64_t> &last_used(int li) const;

  void reset_layer(int li);  // only _staged_state/_next; the rest survives

  struct Metrics {
    uint64_t loads = 0, hits = 0, misses = 0, evictions = 0;
    uint64_t prefetch_submitted = 0, prefetch_hits = 0, prefetch_wasted = 0;
    uint64_t staged_used = 0, staged_fallback = 0, stage_builds = 0;
    uint64_t heap_bytes = 0, heap_peak = 0;
    size_t cache_size = 0, inflight = 0;
  };
  Metrics snapshot() const;
  size_t inflight_size() const {
    std::lock_guard<std::mutex> lk(infl_mu_);
    return inflight_.size();
  }

  static uint64_t key_of(int li, int64_t e) {
    return (uint64_t(li) << 32) | uint64_t(e);
  }

 private:
  struct Layer {
    LayerCfg cfg;
    struct Region {
      const TensorInfo *ti = nullptr;
      int64_t rows = 0, cols = 0;
      uint64_t per = 0;
      uint32_t esize = 4;          // element bytes (u32=4 / bf16=2)
    };
    Region reg[9];
    std::unique_ptr<std::mutex> mu = std::make_unique<std::mutex>();
    StagedState state, next;
    std::deque<std::string> slot_keys;
    std::map<std::string, mx::array> slot_cache;
    std::map<std::string, Asm> asm_cache;
    std::vector<mx::array> zero_rows;
    std::vector<mx::array> incr_tensors;
    mx::array incr_table{0.f};
    std::vector<int64_t> occ;
    std::map<int64_t, std::shared_ptr<Bundle>> incr_bundles;
    // Hot expert keys only; shard mmap holds the bytes. shared_ptr snapshot so
    // pool threads can read keys without UAF.
    struct HotStack {
      std::vector<int64_t> key;
      std::vector<int64_t> sorted;
    };
    std::shared_ptr<const HotStack> hot_backing;
    // Separate from L.mu: build is also called from stage_experts while L.mu is held
    // (non-recursive lock). Writers (load/clear_hot_layer) vs pool-thread build readers.
    std::unique_ptr<std::mutex> backing_mu = std::make_unique<std::mutex>();
    std::vector<int64_t> hot_key;
    std::vector<mx::array> hot_weights;
    uint64_t hot_heap = 0;
    std::vector<mx::array> full_w;
    uint64_t full_heap = 0;
    std::vector<int64_t> hc_order;
    std::unordered_map<int64_t, double> hc_val;
    std::vector<int64_t> last_prefill_topk;
    std::vector<int64_t> last_used;
    int64_t calls = 0;
    // New keys append; existing keys update in place (dict insertion order).
    void bump_count(int64_t e, double decay) {
      auto it = hc_val.find(e);
      if (it == hc_val.end()) { hc_order.push_back(e); hc_val.emplace(e, 1.0); }
      else it->second = it->second * decay + 1.0;
    }
  };

  void stage_incr_locked(Layer &L, const std::vector<int64_t> &uniq);
  int layer_idx(int li) const;

  struct HeapAcc {
    std::atomic<uint64_t> live{0}, peak{0};
    void add(uint64_t b) {
      uint64_t v = live.fetch_add(b) + b;
      uint64_t p = peak.load();
      while (v > p && !peak.compare_exchange_weak(p, v)) {
      }
    }
    void sub(uint64_t b) { live.fetch_sub(b); }
  };

  const ModelFiles *files_;
  struct ShardMaps {
    ~ShardMaps();
    const uint8_t *raw(const std::string &shard_path, uint64_t off, uint64_t len);
    void advise_range(const std::string &shard_path, uint64_t off, uint64_t len);
    void advise_whole(const std::string &shard_path);
    void seq_read(const std::string &shard_path);
    std::mutex mu_;
    std::map<std::string, std::pair<uint8_t *, uint64_t>> maps_;
    std::map<std::string, int> fds_;
  } shards_;

  std::vector<Layer> layers_;
  std::map<int, int> li2idx_;
  SharedExpertCache cache_;
  PrefetchBuffer pbuf_;
  IoPool load_pool_, prefetch_pool_;
  mutable std::mutex infl_mu_;
  std::map<uint64_t, std::shared_future<std::shared_ptr<Bundle>>> inflight_;
  std::shared_ptr<HeapAcc> heap_{std::make_shared<HeapAcc>()};
  std::atomic<uint64_t> loads_{0}, hits_{0}, misses_{0}, psub_{0}, phi_{0}, pwaste_{0},
      sused_{0}, sfallback_{0}, sbuilds_{0};
};

}  // namespace e0n