Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
11.1 KB
Raw Blame
#include "prerouter.hpp"

#include <algorithm>
#include <cmath>
#include <cstring>

#include <mlx/mlx.h>

#include "dense.hpp"

using mlx::core::array;
using mlx::core::Device;

namespace e0n {
namespace {

uint16_t to_f16_bits1(float v) {
  _Float16 h = static_cast<_Float16>(v);
  uint16_t b;
  std::memcpy(&b, &h, 2);
  return b;
}

float f16_bits_to_f(uint16_t b) {
  _Float16 h;
  std::memcpy(&h, &b, 2);
  return static_cast<float>(h);
}

std::vector<float> expand_f16(const std::vector<float> &v) {
  std::vector<float> o(v.size());
  for (size_t i = 0; i < v.size(); i++) o[i] = f16_bits_to_f(to_f16_bits1(v[i]));
  return o;
}

}  // namespace

struct Prerouter::Impl {
  TierRouting reg;
  int64_t hidden = 0, f_in = 0;
  std::vector<int> owners;
  std::vector<float> fc1, fc2, lin;
  bool ready = false;
};

Prerouter::Prerouter() : impl_(std::make_unique<Impl>()) {}
Prerouter::~Prerouter() = default;
Prerouter::Prerouter(Prerouter &&) noexcept = default;
Prerouter &Prerouter::operator=(Prerouter &&) noexcept = default;

int Prerouter::n_heads() const { return impl_ ? static_cast<int>(impl_->owners.size()) : 0; }
int Prerouter::owner(int i) const { return impl_->owners[i]; }

const std::vector<int> &Prerouter::owners() const { return impl_->owners; }
const std::vector<float> &Prerouter::stacked_fc1() const { return impl_->fc1; }
const std::vector<float> &Prerouter::stacked_fc2() const { return impl_->fc2; }
const std::vector<float> &Prerouter::stacked_lin() const { return impl_->lin; }
int64_t Prerouter::stacked_f_in() const { return impl_->f_in; }
int64_t Prerouter::stacked_prh() const { return impl_->reg.prerouter_hidden; }
int64_t Prerouter::stacked_E() const { return impl_->reg.num_experts; }

bool Prerouter::build(const std::vector<HeadW> &heads, const TierRouting &reg, int64_t hidden,
                      Prerouter &out, std::string &err) {
  if (heads.empty()) { err = "prerouter: zero heads"; return false; }
  const int64_t prh = reg.prerouter_hidden, E = reg.num_experts;
  int64_t f_in = -1;
  for (auto &h : heads) {
    if (h.fc1.empty() || h.fc2.empty() || h.lin.empty() ||
        static_cast<int64_t>(h.fc2.size()) != E * prh ||
        static_cast<int64_t>(h.lin.size()) != static_cast<int64_t>(h.fc1.size()) / prh * E) {
      err = "prerouter: header shape mismatch owner=" + std::to_string(h.owner);
      return false;
    }
    int64_t fi = static_cast<int64_t>(h.fc1.size()) / prh;
    if (fi != hidden + 2 * E) {
      err = "prerouter: f_in != hidden+2E (" + std::to_string(fi) + " vs " +
            std::to_string(hidden + 2 * E) + ") owner=" + std::to_string(h.owner);
      return false;
    }
    if (f_in < 0) f_in = fi;
    else if (f_in != fi) { err = "prerouter: f_in disagrees across heads"; return false; }
  }
  for (size_t i = 1; i < heads.size(); i++)
    if (heads[i].owner <= heads[i - 1].owner) { err = "prerouter: owners are not sorted"; return false; }

  auto im = std::make_unique<Prerouter::Impl>();
  im->reg = reg;
  im->hidden = hidden;
  im->f_in = f_in;
  const int O = static_cast<int>(heads.size());
  for (auto &h : heads) im->owners.push_back(h.owner);

  std::vector<float> fc1t(static_cast<size_t>(O) * f_in * prh),
      fc2s(static_cast<size_t>(O) * prh * E), lin_s(static_cast<size_t>(O) * f_in * E);
  for (int i = 0; i < O; i++) {
    const HeadW &h = heads[i];
    for (int64_t r = 0; r < prh; r++)
      for (int64_t c = 0; c < f_in; c++)
        fc1t[(static_cast<size_t>(i) * f_in + c) * prh + r] = h.fc1[r * f_in + c];
    for (int64_t e = 0; e < E; e++) {
      for (int64_t r = 0; r < prh; r++)
        fc2s[(static_cast<size_t>(i) * prh + r) * E + e] = h.fc2[e * prh + r];
      for (int64_t c = 0; c < f_in; c++)
        lin_s[(static_cast<size_t>(i) * f_in + c) * E + e] = h.lin[e * f_in + c];
    }
  }
  im->fc1 = expand_f16(fc1t);
  im->fc2 = expand_f16(fc2s);
  im->lin = expand_f16(lin_s);
  im->ready = true;
  out.impl_ = std::move(im);
  return true;
}

bool Prerouter::load(const ModelFiles &files, const TierRouting &reg, int64_t hidden,
                     Prerouter &out, std::string &err) {
  std::vector<HeadW> heads;
  for (int o = reg.first_owner; o <= reg.last_owner; o++) {
    HeadW h;
    h.owner = o;
    const std::string base = reg.head_file;
    DenseF32 fc1, fc2, lin;
    std::string k1 = "layers." + std::to_string(o) + ".fc1.weight";
    std::string k2 = "layers." + std::to_string(o) + ".fc2.weight";
    std::string k3 = "layers." + std::to_string(o) + ".linear_init.weight";
    if (!read_dense_f32(files, k1, fc1, err) || !read_dense_f32(files, k2, fc2, err) ||
        !read_dense_f32(files, k3, lin, err)) {
      if (err.find("missing tensor") == 0) err += " (header file " + base + " or missing from the directory)";
      return false;
    }
    if (fc1.rows != reg.prerouter_hidden || fc2.cols != reg.prerouter_hidden ||
        fc2.rows != reg.num_experts || lin.rows != reg.num_experts || fc1.cols != lin.cols) {
      err = "prerouter: header shape does not match registry owner=" + std::to_string(o);
      return false;
    }
    h.fc1 = std::move(fc1.v);
    h.fc2 = std::move(fc2.v);
    h.lin = std::move(lin.v);
    heads.push_back(std::move(h));
  }
  return build(heads, reg, hidden, out, err);
}

bool Prerouter::predict(const std::vector<Input> &in, std::vector<Selection> &out_sel,
                        std::vector<std::vector<double>> *out_logits, std::string &err) {
  if (!impl_ || !impl_->ready) { err = "prerouter: not built"; return false; }
  const int O = n_heads();
  if (static_cast<int>(in.size()) != O) { err = "prerouter: head count does not match input count"; return false; }
  const auto &reg = impl_->reg;
  const int64_t prh = reg.prerouter_hidden, E = reg.num_experts, f_in = impl_->f_in;

  std::vector<float> feats(static_cast<size_t>(O) * f_in, 0.f);
  for (int i = 0; i < O; i++) {
    if (!in[i].hidden) { err = "prerouter: missing header hidden input owner=" + std::to_string(owner(i)); return false; }
    float *row = feats.data() + static_cast<size_t>(i) * f_in;
    std::memcpy(row, in[i].hidden, sizeof(float) * impl_->hidden);
    for (int64_t e : in[i].cur_exec)
      if (e >= 0 && e < E) row[impl_->hidden + e] = 1.f;
    for (int64_t e : in[i].prev_exec)
      if (e >= 0 && e < E) row[impl_->hidden + E + e] = 1.f;
  }

  try {
    auto wrap = [](void *p, mlx::core::Shape shape, mlx::core::Dtype dt) {
      return array(p, std::move(shape), dt, [](void *) {});
    };
    auto a_feats = wrap(feats.data(), {O, 1, static_cast<int32_t>(f_in)}, mlx::core::float32);
    auto a_fc1 = wrap(impl_->fc1.data(),
                      {O, static_cast<int32_t>(f_in), static_cast<int32_t>(prh)},
                      mlx::core::float32);
    auto a_fc2 = wrap(impl_->fc2.data(),
                      {O, static_cast<int32_t>(prh), static_cast<int32_t>(E)},
                      mlx::core::float32);
    auto a_lin = wrap(impl_->lin.data(),
                      {O, static_cast<int32_t>(f_in), static_cast<int32_t>(E)},
                      mlx::core::float32);
    const auto f16 = mlx::core::float16;
    auto qf = mlx::core::astype(a_feats, f16);
    auto h = mlx::core::matmul(qf, mlx::core::astype(a_fc1, f16));  // [O,1,prh] f16
    auto gel = h * 0.5f * (1.0f + mlx::core::erf(h / static_cast<float>(std::sqrt(2.0))));
    auto logits = mlx::core::astype(
        mlx::core::matmul(gel, mlx::core::astype(a_fc2, f16)) +
            mlx::core::matmul(qf, mlx::core::astype(a_lin, f16)),
        mlx::core::float32);
    auto host = mlx::core::copy(logits, mlx::core::default_stream(Device::cpu));
    mlx::core::eval(host);
    const float *lp = host.data<float>();
    out_sel.assign(O, Selection{});
    if (out_logits) out_logits->assign(O, {});
    for (int i = 0; i < O; i++) {
      const double *rowp = nullptr;
      std::vector<double> drow(E);
      for (int64_t e = 0; e < E; e++) drow[e] = lp[static_cast<size_t>(i) * E + e];
      rowp = drow.data();
      if (out_logits) (*out_logits)[i] = drow;
      out_sel[i] = reg.family == RouterFamily::kSoftmaxTopk
                       ? select_softmax_topk(rowp, static_cast<int>(E), reg.top_k,
                                             reg.norm_topk_prob)
                       : select_sigmoid_group(rowp, nullptr, static_cast<int>(E), reg.top_k,
                                              reg.n_group, reg.topk_group, reg.routed_scaling,
                                              reg.norm_topk_prob);
    }
  } catch (const std::exception &ex) {
    err = std::string("prerouter mlx: ") + ex.what();
    return false;
  }
  return true;
}

// ---------------- PrerouterState ----------------

Step live_next_step(const PrerouterState &st) {
  Step plan;
  for (int l = st.reg.first_head_consumer; l <= st.reg.last_head_consumer; l++)
    if (!st.pred[l].inds.empty())
      for (auto e : st.pred[l].inds) plan.experts.emplace_back(l, e);
  return plan;
}

void PrerouterState::init(const TierRouting &r) {
  reg = r;
  cap_hidden.assign(r.num_layers, {});
  cap_exec.assign(r.num_layers, {});
  prev_exec.assign(r.num_layers, {});
  pred.assign(r.num_layers, Selection{});
  pred_avail.assign(r.num_layers, false);
  step = 0;
}

void PrerouterState::capture(int layer, std::vector<float> m_in, std::vector<int64_t> exec) {
  cap_hidden[layer] = std::move(m_in);
  std::sort(exec.begin(), exec.end());
  cap_exec[layer] = std::move(exec);
}

bool PrerouterState::run_step_boundary(Prerouter &pr,
                                       std::vector<std::vector<double>> *diag_logits,
                                       std::string &err) {
  const int O = pr.n_heads();
  for (int l = reg.first_head_consumer; l <= reg.last_head_consumer; l++) pred_avail[l] = false;
  std::vector<Prerouter::Input> in(O);
  for (int i = 0; i < O; i++) {
    int own = pr.owner(i);
    if (cap_hidden[own].empty()) continue;
    in[i].hidden = cap_hidden[own].data();
    in[i].cur_exec = cap_exec[own];
    in[i].prev_exec = prev_exec[own];
  }
  std::vector<Prerouter::Input> compact;
  std::vector<int> idx_of;
  for (int i = 0; i < O; i++)
    if (in[i].hidden) { compact.push_back(in[i]); idx_of.push_back(i); }
  if (compact.empty()) { swap(); return true; }

  std::vector<Selection> sel;
  std::vector<std::vector<double>> lg;
  if (!pr.predict(compact, sel, diag_logits ? &lg : nullptr, err)) return false;
  if (diag_logits) *diag_logits = std::move(lg);
  for (size_t j = 0; j < idx_of.size(); j++) {
    int own = pr.owner(idx_of[j]);
    int consumer = own + 1;
    if (consumer >= reg.first_head_consumer && consumer <= reg.last_head_consumer) {
      pred[consumer] = sel[j];
      pred_avail[consumer] = true;
    }
  }
  if (diag_logits) {
  }
  swap();
  return true;
}

void PrerouterState::swap() {
  prev_exec = cap_exec;
  for (auto &v : cap_hidden) v.clear();
  for (auto &v : cap_exec) v.clear();
  step++;
}

bool PrerouterState::take_pred(int layer, Selection &out) {
  if (layer < 0 || layer >= reg.num_layers || !pred_avail[layer] || pred[layer].inds.empty())
    return false;
  out = pred[layer];
  pred_avail[layer] = false;
  return true;
}

void PrerouterState::reset() {
  for (auto &v : cap_hidden) v.clear();
  for (auto &v : cap_exec) v.clear();
  for (auto &v : prev_exec) v.clear();
  for (auto &p : pred) p = Selection{};
  std::fill(pred_avail.begin(), pred_avail.end(), false);
  step = 0;
}

}  // namespace e0n