Skip to content

armgit / Edge0

Public Imported from GitHub
0

Run 35B MoE models in ~2.5 GB of RAM. Weights stream from SSD; a trained router predicts loads a step ahead. Mac, iPhone, Android, Windows supported

Python
33.9 KB
Raw Blame
#include "engine-abi.h"

#include <algorithm>
#include <cstring>
#include <cassert>
#include <cstdlib>
#include <fstream>
#include <memory>
#include <mutex>
#include <string>
#include <vector>

#include <sys/stat.h>
#include <dirent.h>

#include "memstat.hpp"
#include "fwd_common.hpp"
#include "safio.hpp"
#include "expertmap.hpp"
#include "hotstack.hpp"
#include "expertio.hpp"
#include "net35.hpp"
#include "net8b.hpp"
#include "genstream.hpp"
#include "prerouter.hpp"
#include "prefixcache.hpp"
#include "sampling.hpp"

#include <nlohmann/json.hpp>

extern "C" {
typedef struct ETok ETok;
void *etok_create(const char *model_dir);
void etok_destroy(void *t);
int etok_last_error(char *buf, size_t buf_len, size_t *written);
int etok_tokenize(void *t, const char *text, int32_t *ids_out, size_t ids_cap,
                  size_t *written);
int etok_decode(void *t, const int32_t *ids, size_t n_ids, char *buf, size_t buf_len,
                size_t *written);
int etok_render(void *t, const char *ctx_json, char *buf, size_t buf_len, size_t *written);
}

namespace {

std::string esc(const std::string &s) {
  std::string o;
  for (char c : s) {
    if (c == '"' || c == '\\') { o += '\\'; o += c; }
    else if (c == '\n') o += "\\n";
    else o += c;
  }
  return o;
}

E0Status json_out(const std::string &json, char *buf, size_t buf_len, size_t *written) {
  if (written) *written = json.size();
  if (buf == nullptr) return E0_OK;
  if (buf_len < json.size()) return E0_ERR_BUFFER;
  std::memcpy(buf, json.data(), json.size());
  return E0_OK;
}

bool is_dir(const char *p) {
  struct stat st;
  return p && stat(p, &st) == 0 && S_ISDIR(st.st_mode);
}

}  // namespace

std::string etok_err_json() {
  size_t w = 0;
  etok_last_error(nullptr, 0, &w);
  std::string msg(w, '\0');
  if (w) etok_last_error(msg.data(), w, &w);
  return std::string("{\"error\":\"E-INTERNAL\",\"message\":\"") + esc(msg) + "\"}";
}

namespace e0n { namespace gen { class StreamBase; } }

struct E0Engine {
  std::string tier;
  std::string model_dir;
  std::string config_json;
  std::mutex mu;
  std::string last_error;
  std::shared_ptr<e0n::ModelFiles> files;
  std::shared_ptr<e0n::ExpertMap> experts;
  std::unique_ptr<e0n::PrefixCache> prefix_cache;
  std::string model_identity;
  std::unique_ptr<e0n::Net35> net35;
  std::unique_ptr<e0n::State35> st35;
  std::unique_ptr<e0n::Net8b> net8b;
  std::unique_ptr<e0n::State8b> st8b;
  std::vector<e0n::RouteRec> last_recs;
  bool has_recs = false;
  std::vector<int32_t> eos_ids;
  e0n::gen::StreamBase *active = nullptr;
};

struct CachedState35 {
  e0n::State35 state;
  std::vector<float> logits;
};

struct CachedState8 {
  e0n::State8b state;
  std::vector<float> logits;
};

namespace {

mlx::core::array snapshot_array(const mlx::core::array &a) {
  auto out = mlx::core::copy(a);
  mlx::core::eval(out);
  return out;
}

template <typename T>
std::vector<mlx::core::array> snapshot_arrays(const std::vector<mlx::core::array> &src) {
  std::vector<mlx::core::array> out;
  out.reserve(src.size());
  for (const auto &a : src) out.push_back(snapshot_array(a));
  return out;
}

std::shared_ptr<CachedState35> capture_state35(const e0n::State35 &src,
                                                const std::vector<float> &logits) {
  auto out = std::make_shared<CachedState35>();
  out->state.pr = src.pr;
  out->state.conv = snapshot_arrays<float>(src.conv);
  out->state.recur = snapshot_arrays<float>(src.recur);
  out->state.kc = snapshot_arrays<float>(src.kc);
  out->state.vc = snapshot_arrays<float>(src.vc);
  out->state.is_linear = src.is_linear;
  out->state.position = src.position;
  out->state.started = src.started;
  out->logits = logits;
  return out;
}

std::shared_ptr<CachedState8> capture_state8(const e0n::State8b &src,
                                               const std::vector<float> &logits) {
  auto out = std::make_shared<CachedState8>();
  out->state.pr = src.pr;
  out->state.cq = snapshot_arrays<float>(src.cq);
  out->state.ck = snapshot_arrays<float>(src.ck);
  out->state.cv = snapshot_arrays<float>(src.cv);
  out->state.ssm = snapshot_arrays<float>(src.ssm);
  out->state.kc = snapshot_arrays<float>(src.kc);
  out->state.vc = snapshot_arrays<float>(src.vc);
  out->state.is_mla = src.is_mla;
  out->state.position = src.position;
  out->state.started = src.started;
  out->logits = logits;
  return out;
}

void restore_state35(e0n::State35 &dst, const e0n::State35 &src) {
  dst.pr = src.pr;
  dst.conv = snapshot_arrays<float>(src.conv);
  dst.recur = snapshot_arrays<float>(src.recur);
  dst.kc = snapshot_arrays<float>(src.kc);
  dst.vc = snapshot_arrays<float>(src.vc);
  dst.is_linear = src.is_linear;
  dst.position = src.position;
  dst.started = src.started;
}

void restore_state8(e0n::State8b &dst, const e0n::State8b &src) {
  dst.pr = src.pr;
  dst.cq = snapshot_arrays<float>(src.cq);
  dst.ck = snapshot_arrays<float>(src.ck);
  dst.cv = snapshot_arrays<float>(src.cv);
  dst.ssm = snapshot_arrays<float>(src.ssm);
  dst.kc = snapshot_arrays<float>(src.kc);
  dst.vc = snapshot_arrays<float>(src.vc);
  dst.is_mla = src.is_mla;
  dst.position = src.position;
  dst.started = src.started;
}

uint64_t state_bytes(const e0n::State35 &s) {
  uint64_t n = sizeof(s);
  for (const auto *v : {&s.conv, &s.recur, &s.kc, &s.vc})
    for (const auto &a : *v) n += a.nbytes();
  for (const auto &v : s.pr.cap_hidden) n += v.size() * sizeof(float);
  for (const auto &v : s.pr.cap_exec) n += v.size() * sizeof(int64_t);
  for (const auto &v : s.pr.prev_exec) n += v.size() * sizeof(int64_t);
  return n;
}

uint64_t state_bytes(const e0n::State8b &s) {
  uint64_t n = sizeof(s);
  for (const auto *v : {&s.cq, &s.ck, &s.cv, &s.ssm, &s.kc, &s.vc})
    for (const auto &a : *v) n += a.nbytes();
  for (const auto &v : s.pr.cap_hidden) n += v.size() * sizeof(float);
  for (const auto &v : s.pr.cap_exec) n += v.size() * sizeof(int64_t);
  for (const auto &v : s.pr.prev_exec) n += v.size() * sizeof(int64_t);
  return n;
}

uint64_t snapshot_bytes(const CachedState35 &s) {
  return state_bytes(s.state) + s.logits.size() * sizeof(float);
}

uint64_t snapshot_bytes(const CachedState8 &s) {
  return state_bytes(s.state) + s.logits.size() * sizeof(float);
}

}  // namespace

struct E0Tokenizer {
  std::unique_ptr<void, void (*)(void *)> raw;
  explicit E0Tokenizer(void *h)
      : raw(h, [](void *p) { if (p) etok_destroy(p); }) {}
};

static std::string g_create_error;
static std::mutex g_tok_err_mu;

#include <unordered_map>

static std::mutex g_pool_mu;
static std::unordered_map<E0Engine *, std::unique_ptr<E0Engine>> g_engines;
static std::unordered_map<E0Tokenizer *, std::unique_ptr<E0Tokenizer>> g_toks;
static std::unordered_map<E0Stream *, std::unique_ptr<e0n::gen::StreamBase>> g_streams;

#include "genstream.hpp"

template <class MapT, class PtrT>
static PtrT pool_new(MapT &m, PtrT p) {
  std::lock_guard<std::mutex> g(g_pool_mu);
  using V = typename MapT::mapped_type::element_type;
  m.emplace(p, std::unique_ptr<V>(static_cast<V *>((void *)p)));
  return p;
}
template <class MapT, class PtrT>
static void pool_free(MapT &m, PtrT p) {
  std::lock_guard<std::mutex> g(g_pool_mu);
  m.erase(p);
}

namespace {
uint64_t logits_hash(const std::vector<float> &v) {
  uint64_t h = 1469598103934665603ull;
  const uint8_t *p = reinterpret_cast<const uint8_t *>(v.data());
  for (size_t i = 0; i < v.size() * sizeof(float); i++) { h ^= p[i]; h *= 1099511628211ull; }
  return h;
}
}  // namespace

template <class NetT, class StateT>
E0Status test_forward_run(std::unique_ptr<NetT> &net, std::unique_ptr<StateT> &st, E0Engine *h,
                          const std::vector<int32_t> &tok, uint32_t steps, char *buf,
                          size_t buf_len, size_t *written) {
  std::string err;
  std::vector<float> lg;
  std::vector<e0n::RouteRec> recs;
  if (!net->prefill(tok, *st, lg, recs, err)) {
    h->last_error = "{\"error\":\"E-MODEL-INVALID\",\"message\":\"" + esc(err) + "\"}";
    return E0_ERR_INTERNAL;
  }
  h->last_recs = recs;
  h->has_recs = true;
  auto argmax = [&](const std::vector<float> &v) {
    size_t b = 0;
    for (size_t i = 1; i < v.size(); i++) if (v[i] > v[b]) b = i;
    return static_cast<int32_t>(b);
  };
  std::string json = "{\"prefill_logits_finite\":" +
                     std::string(std::all_of(lg.begin(), lg.end(), [](float x) { return std::isfinite(x); }) ? "true" : "false") +
                     ",\"lh\":" + std::to_string(logits_hash(lg)) +
                     ",\"argmax\":" + std::to_string(argmax(lg)) + ",\"steps\":[";
  int32_t cur = argmax(lg);
  for (uint32_t s = 0; s < steps; s++) {
    if (!net->step(cur, *st, lg, recs, err)) {
      h->last_error = "{\"error\":\"E-MODEL-INVALID\",\"message\":\"" + esc(err) + "\"}";
      return E0_ERR_INTERNAL;
    }
    h->last_recs = recs;
    h->has_recs = true;
    bool fin = std::all_of(lg.begin(), lg.end(), [](float x) { return std::isfinite(x); });
    cur = argmax(lg);
    json += (s ? ",{" : "{") + std::string("\"finite\":") + (fin ? "true" : "false") +
            ",\"lh\":" + std::to_string(logits_hash(lg)) +
            ",\"argmax\":" + std::to_string(cur) + "}";
  }
  json += "]}";
  return json_out(json, buf, buf_len, written);
}

static void sweep_terminal(const char *what) {
  (void)what;
  std::lock_guard<std::mutex> pg(g_pool_mu);
  for (auto it = g_streams.begin(); it != g_streams.end();) {
    if (it->second && it->second->terminal()) it = g_streams.erase(it);
    else ++it;
  }
}

extern "C" {

#ifndef E0N_VERSION
#define E0N_VERSION "0.1.0"
#endif

uint32_t e0_abi_version(void) { return E0_ABI_VERSION; }
const char *e0_engine_version(void) { return E0N_VERSION; }

E0Engine *e0_engine_create(const char *tier_id, const char *model_dir,
                           const char *config_json) {
  auto note = [](const std::string &e) { g_create_error = e; };
  if (!tier_id || !*tier_id) {
    note("{\"error\":\"E-MODEL-INVALID\",\"message\":\"tier_id is empty\"}");
    return nullptr;
  }
  auto files = std::make_shared<e0n::ModelFiles>();
  std::string err;
  if (!files->open(model_dir ? model_dir : "", err)) {
    note(std::string("{\"error\":\"E-MODEL-PATH\",\"message\":\"") + esc(err) + "\"}");
    return nullptr;
  }
  auto experts = std::make_shared<e0n::ExpertMap>();
  if (!e0n::ExpertMap::build(*files, *experts, err)) {
    note(std::string("{\"error\":\"E-MODEL-LAYOUT\",\"message\":\"") + esc(err) + "\"}");
    return nullptr;
  }
  uint64_t budget = 2ull << 30;
  int io_threads = 3;
  try {
    auto cj = nlohmann::json::parse(config_json ? config_json : "{}", nullptr, false);
    if (!cj.is_discarded()) {
      if (cj.contains("cache_budget_bytes") && cj["cache_budget_bytes"].is_number_unsigned())
        budget = cj["cache_budget_bytes"].get<uint64_t>();
      if (cj.contains("io_threads") && cj["io_threads"].is_number_integer())
        io_threads = std::max(1, cj["io_threads"].get<int>());
    }
  } catch (...) {}
  auto *h = new E0Engine();
  h->tier = tier_id;
  h->model_dir = model_dir ? model_dir : "";
  h->config_json = config_json ? config_json : "{}";
  h->files = std::move(files);
  h->experts = std::move(experts);
  e0n::PrefixCacheConfig pcc;
  pcc.enabled = true;
  std::string identity = std::string(tier_id) + "|" + (model_dir ? model_dir : "|") +
                         "|abi=" + std::to_string(E0_ABI_VERSION);
  try {
    auto cj = nlohmann::json::parse(h->config_json, nullptr, false);
    if (!cj.is_discarded()) {
      if (cj.contains("model_identity")) identity = cj["model_identity"].dump();
      if (cj.contains("prefix_cache") && cj["prefix_cache"].is_object()) {
        const auto &pc = cj["prefix_cache"];
        if (pc.contains("enabled") && pc["enabled"].is_boolean())
          pcc.enabled = pc["enabled"].get<bool>();
        if (pc.contains("budget_bytes") && pc["budget_bytes"].is_number_unsigned())
          pcc.budget_bytes = pc["budget_bytes"].get<uint64_t>();
        if (pc.contains("min_prefix_tokens") && pc["min_prefix_tokens"].is_number_unsigned())
          pcc.min_prefix_tokens = std::max<uint32_t>(1, pc["min_prefix_tokens"].get<uint32_t>());
      }
    }
  } catch (...) {
    identity = std::string(tier_id) + "|" + (model_dir ? model_dir : "|") +
               "|abi=" + std::to_string(E0_ABI_VERSION);
  }
  h->model_identity = std::move(identity);
  h->prefix_cache = std::make_unique<e0n::PrefixCache>(pcc);
  try {
    auto cj = nlohmann::json::parse(h->config_json, nullptr, false);
    if (!cj.is_discarded() && cj.contains("forward") && cj["forward"].is_string()) {
      const std::string fwd = cj["forward"].get<std::string>();
      if (fwd == "35b" || fwd == "8b") {
        const std::string want_tier = "edge0-" + fwd;
        if (h->tier != want_tier) {
          pool_free(g_engines, h);
          note("{\"error\":\"E-MODEL-INVALID\",\"message\":\"forward=" + fwd +
               " expected tier=" + want_tier + "\"}");
          return nullptr;
        }
        std::shared_ptr<const e0n::lora::Bundle> lo;
        if (cj.contains("lora") && cj["lora"].is_boolean() && cj["lora"].get<bool>()) {
          e0n::TierRouting treg;
          if (!e0n::tier_routing(h->tier, treg)) {
            pool_free(g_engines, h);
            note("{\"error\":\"E-MODEL-INVALID\",\"message\":\"tier missing from registry\"}");
            return nullptr;
          }
          auto b = std::make_shared<e0n::lora::Bundle>();
          std::string lerr;
          if (!b->load_file(h->model_dir + "/" + treg.lora_file, fwd == "8b",
                            treg.num_layers, lerr)) {
            pool_free(g_engines, h);
            const bool missing = lerr.rfind("lora file unreadable", 0) == 0;
            note(std::string("{\"error\":\"") + (missing ? "E-MODEL-INVALID" : "E-MODEL-LAYOUT") +
                 "\",\"message\":\"lora: " + esc(lerr) + "\"}");
            return nullptr;
          }
          lo = b;
        }
        std::string ferr;
        if (fwd == "35b") {
          h->net35 = e0n::Net35::create(h->files, h->experts, ferr, lo);
          if (!h->net35) {
            pool_free(g_engines, h);
            note(std::string("{\"error\":\"E-MODEL-LAYOUT\",\"message\":\"forward ctx: ") +
                 esc(ferr) + "\"}");
            return nullptr;
          }
          h->st35 = std::make_unique<e0n::State35>();
          h->net35->init_state(*h->st35);
        } else {
          h->net8b = e0n::Net8b::create(h->files, h->experts, ferr, lo);
          if (!h->net8b) {
            pool_free(g_engines, h);
            note(std::string("{\"error\":\"E-MODEL-LAYOUT\",\"message\":\"forward ctx: ") +
                 esc(ferr) + "\"}");
            return nullptr;
          }
          h->st8b = std::make_unique<e0n::State8b>();
          h->net8b->init_state(*h->st8b);
        }
        std::ifstream cf(h->model_dir + "/config.json");
        if (cf) {
          auto cj2 = nlohmann::json::parse(cf, nullptr, false);
          if (!cj2.is_discarded() && cj2.contains("eos_token_id")) {
            const auto &x = cj2["eos_token_id"];
            if (x.is_number_integer()) {
              h->eos_ids.push_back(x.get<int32_t>());
            } else if (x.is_array()) {
              for (const auto &e : x)
                if (e.is_number_integer()) h->eos_ids.push_back(e.get<int32_t>());
            }
          }
        }
      }
    }
  } catch (const std::exception &e) {
    note(std::string("{\"error\":\"E-INTERNAL\",\"message\":\"create: ") +
         esc(e.what()) + "\"}");
    delete h;
    return nullptr;
  }
  return pool_new(g_engines, h);
}

E0Stream *e0_generate_begin(E0Engine *h, const int32_t *prompt_tokens, uint32_t n_tokens,
                            const char *params_json) {
  if (!h) return nullptr;
  {
    std::lock_guard<std::mutex> pg(g_pool_mu);
    if (!g_engines.count(h)) return nullptr;
  }
  sweep_terminal("begin");
  std::lock_guard<std::mutex> g(h->mu);
  e0n::sampling::TierDefaults td = e0n::sampling::tier_defaults(h->tier);
  auto p = e0n::sampling::parse_params(params_json ? params_json : "{}", td, h->eos_ids);
  bool transport_only = false;
  if (p.teacher_tokens.empty()) {
    auto pv0 = nlohmann::json::parse(params_json ? params_json : "{}", nullptr, false);
    if (!pv0.is_discarded() && pv0.contains("replay_tokens") && pv0["replay_tokens"].is_array()) {
      for (const auto &it : pv0["replay_tokens"])
        if (it.is_number_integer()) p.teacher_tokens.push_back(it.get<int32_t>());
      transport_only = !p.teacher_tokens.empty();
    }
  }
  if (params_json) {
    auto pv = nlohmann::json::parse(params_json, nullptr, false);
    if (!pv.is_discarded() && pv.contains("stop") && pv["stop"].is_array()) {
      for (const auto &it : pv["stop"]) {
        if (!it.is_string() || it.get_ref<const std::string &>().empty()) continue;
        void *tk = etok_create(h->model_dir.c_str());
        if (!tk) {
          h->last_error = etok_err_json();
          return nullptr;
        }
        std::vector<int32_t> ids;
        size_t w = 0;
        const std::string &str = it.get_ref<const std::string &>();
        if (etok_tokenize(tk, str.c_str(), nullptr, 0, &w) == 0) {
          ids.resize(w);
          if (w) etok_tokenize(tk, str.c_str(), ids.data(), w, &w);
        }
        etok_destroy(tk);
        p.stop_seqs.push_back(std::move(ids));
      }
    }
  }
  std::vector<int32_t> prompt(prompt_tokens, prompt_tokens + n_tokens);
  const size_t prompt_len = prompt.size();
  e0n::gen::StreamBase *ns;
#ifdef E0N_TEST_SCAFFOLD
  if (const char *sc = getenv("E0N_SCRIPT")) {
    std::vector<std::vector<float>> script;
    const int n = atoi(sc);
    for (int i = 0; i < n; i++) {
      std::vector<float> lg(8, -10.f);
      lg[i % 8] = 5.f;
      script.push_back(std::move(lg));
    }
    ns = new e0n::gen::ScriptedStream(std::move(prompt), std::move(p), std::move(script));
    ns->set_owner(h);
    if (h->active) h->active->seal();
    h->active = ns;
    return pool_new(g_streams, (E0Stream *)ns);
  }
#endif
  if (transport_only && (h->net35 || h->net8b)) {
    ns = new e0n::gen::StreamBase(std::move(prompt), std::move(p));
    ns->set_transport();
    ns->set_owner(h);
    if (h->active) h->active->seal();
    h->active = ns;
    return pool_new(g_streams, (E0Stream *)ns);
  }
  if (!h->net35 && !h->net8b) {
    const bool empty_turn = p.teacher_tokens.empty() && prompt.empty();
    ns = new e0n::gen::StreamBase(std::move(prompt), std::move(p));
    ns->set_transport();
    ns->set_owner(h);
    if (empty_turn) ns->set_empty_done();
    if (h->active) h->active->seal();
    h->active = ns;
    return pool_new(g_streams, (E0Stream *)ns);
  }
  std::vector<int32_t> prefill_prompt = prompt;
  size_t cached_tokens = 0;
  if (h->net35) {
    auto hit = h->prefix_cache->lookup(h->model_identity, prompt);
    if (hit.snapshot) {
      try {
        auto snap = std::static_pointer_cast<CachedState35>(hit.snapshot);
        restore_state35(*h->st35, snap->state);
        cached_tokens = hit.matched_tokens;
        prefill_prompt.assign(prompt.begin() + static_cast<std::ptrdiff_t>(cached_tokens), prompt.end());
      } catch (const std::exception &e) {
        h->prefix_cache->clear();
        h->last_error = std::string("{\"error\":\"E-INTERNAL\",\"message\":\"") +
                        esc(e.what()) + "\"}";
        return nullptr;
      }
    } else {
      h->net35->reset_state(*h->st35);
    }
    ns = new e0n::gen::Stream<e0n::Net35, e0n::State35>(
        h->net35.get(), h->st35.get(),
        [net = h->net35.get()](const e0n::Step &plan) { net->prefetch(plan); }, h,
        std::move(prompt), std::move(p));
    ns->set_prefill_prompt(std::move(prefill_prompt));
    if (cached_tokens == prompt_len && hit.snapshot) {
      auto snap = std::static_pointer_cast<CachedState35>(hit.snapshot);
      ns->set_cached_logits(snap->logits);
    }
    ns->set_on_prefill([h](const std::vector<int32_t> &full, const std::vector<float> &logits) {
      try {
        auto snap = capture_state35(*h->st35, logits);
        h->prefix_cache->put(h->model_identity, full, snap, snapshot_bytes(*snap));
      } catch (const std::exception &e) {
        h->last_error = std::string("{\"error\":\"E-INTERNAL\",\"message\":\"") +
                        esc(e.what()) + "\"}";
      }
    });
    ns->set_on_complete([h](const std::vector<int32_t> &full, const std::vector<float> &logits) {
      try {
        auto snap = capture_state35(*h->st35, logits);
        h->prefix_cache->put(h->model_identity, full, snap, snapshot_bytes(*snap));
      } catch (const std::exception &e) {
        h->last_error = std::string("{\"error\":\"E-INTERNAL\",\"message\":\"") +
                        esc(e.what()) + "\"}";
      }
    });
  } else {
    auto hit = h->prefix_cache->lookup(h->model_identity, prompt);
    if (hit.snapshot) {
      try {
        auto snap = std::static_pointer_cast<CachedState8>(hit.snapshot);
        restore_state8(*h->st8b, snap->state);
        cached_tokens = hit.matched_tokens;
        prefill_prompt.assign(prompt.begin() + static_cast<std::ptrdiff_t>(cached_tokens), prompt.end());
      } catch (const std::exception &e) {
        h->prefix_cache->clear();
        h->last_error = std::string("{\"error\":\"E-INTERNAL\",\"message\":\"") +
                        esc(e.what()) + "\"}";
        return nullptr;
      }
    } else {
      h->net8b->reset_state(*h->st8b);
    }
    ns = new e0n::gen::Stream<e0n::Net8b, e0n::State8b>(
        h->net8b.get(), h->st8b.get(),
        [net = h->net8b.get()](const e0n::Step &plan) { net->prefetch(plan); }, h,
        std::move(prompt), std::move(p));
    ns->set_prefill_prompt(std::move(prefill_prompt));
    if (cached_tokens == prompt_len && hit.snapshot) {
      auto snap = std::static_pointer_cast<CachedState8>(hit.snapshot);
      ns->set_cached_logits(snap->logits);
    }
    ns->set_on_prefill([h](const std::vector<int32_t> &full, const std::vector<float> &logits) {
      try {
        auto snap = capture_state8(*h->st8b, logits);
        h->prefix_cache->put(h->model_identity, full, snap, snapshot_bytes(*snap));
      } catch (const std::exception &e) {
        h->last_error = std::string("{\"error\":\"E-INTERNAL\",\"message\":\"") +
                        esc(e.what()) + "\"}";
      }
    });
    ns->set_on_complete([h](const std::vector<int32_t> &full, const std::vector<float> &logits) {
      try {
        auto snap = capture_state8(*h->st8b, logits);
        h->prefix_cache->put(h->model_identity, full, snap, snapshot_bytes(*snap));
      } catch (const std::exception &e) {
        h->last_error = std::string("{\"error\":\"E-INTERNAL\",\"message\":\"") +
                        esc(e.what()) + "\"}";
      }
    });
  }
  h->prefix_cache->set_last_prefill_tokens(prompt_len - cached_tokens);
  if (h->active) h->active->seal();
  h->active = ns;
  return pool_new(g_streams, (E0Stream *)ns);
}

namespace {
e0n::gen::StreamBase *checked_stream(E0Engine *h, E0Stream *stream) {
  if (!h || !stream) return nullptr;
  std::lock_guard<std::mutex> pg(g_pool_mu);
  if (!g_engines.count(h)) return nullptr;
  auto it = g_streams.find(stream);
  if (it == g_streams.end() || !it->second || !it->second->owner_of(h)) return nullptr;
  return it->second.get();
}
}  // namespace

E0Status e0_next(E0Engine *h, E0Stream *stream, E0TokenOut *out) {
  auto *st = checked_stream(h, stream);
  if (!st) return E0_ERR_INVALID;
  std::lock_guard<std::mutex> g(st->mu);
  if (out) {
    if (out->size < sizeof(E0TokenOut)) return E0_ERR_INVALID;
    out->token = 0;
    out->flags = 0;
  }
  int32_t tok = 0;
  const bool got = st->next_token(tok);
  if (got) {
    if (out) out->token = tok;
    return E0_NOTIFIED;
  }
  const bool errored = st->errored();
  const std::string em = st->err_msg();
  {
    std::lock_guard<std::mutex> pg(g_pool_mu);
    if (st->owner()) {
      st->owner()->last_recs = st->last_recs();
      st->owner()->has_recs = true;
      if (errored) {
        st->owner()->last_error =
            "{\"error\":\"E-INTERNAL\",\"message\":" + esc(em) + "}";
        if (st->owner()->active == st) st->owner()->active = nullptr;
        g_streams.erase(stream);
      }
    }
  }
  if (errored) return E0_ERR_INTERNAL;
  if (out) out->flags = E0_TOKEN_FINISHED;
  return E0_NOTIFIED;
}

E0Status e0_cancel(E0Engine *h, E0Stream *stream) {
  if (!h) return E0_ERR_INVALID;
  auto *st = checked_stream(h, stream);
  if (!st) return E0_OK;
  std::lock_guard<std::mutex> g(st->mu);
  st->cancel();
  return E0_OK;
}

E0Status e0_stats(E0Engine *h, char *buf, size_t buf_len, size_t *written) {
  if (!h) return E0_ERR_INVALID;
  e0n::MemSample m = e0n::mem_sample();
  e0n::ExpertIo::Metrics c{};
  if (h->net35) c = h->net35->io_metrics();
  else if (h->net8b) c = h->net8b->io_metrics();
  std::string route = "{\"route_source_by_layer\":{";
  {
    std::lock_guard<std::mutex> g(h->mu);
    if (h->has_recs) {
      for (size_t i = 0; i < h->last_recs.size(); i++) {
        if (i) route += ",";
        route += "\"" + std::to_string(h->last_recs[i].layer) + "\":\"" +
                 (h->last_recs[i].predicted ? "predicted" : "routed") + "\"";
      }
    }
  }
  route += "}";
  auto cx = e0n::exec_counters().snapshot();
  std::string diag = ",\"diag\":{\"qmm_calls\":" + std::to_string(cx.qmm_calls) +
                     ",\"qmm_experts\":" + std::to_string(cx.qmm_experts) +
                     ",\"qmm_weight_bytes\":" + std::to_string(cx.qmm_weight_bytes) +
                     ",\"host_syncs\":" + std::to_string(cx.host_syncs) +
                     ",\"host_wraps\":" + std::to_string(cx.host_wraps) +
                     ",\"leases\":" + std::to_string(cx.leases) + "}";
  const auto pc = h->prefix_cache ? h->prefix_cache->metrics() : e0n::PrefixCacheMetrics{};
  std::string prefix = ",\"prefix_cache\":{\"enabled\":" +
                       std::string(pc.enabled ? "true" : "false") +
                       ",\"budget_bytes\":" + std::to_string(pc.budget_bytes) +
                       ",\"min_prefix_tokens\":" + std::to_string(pc.min_prefix_tokens) +
                       ",\"hits\":" + std::to_string(pc.hits) +
                       ",\"misses\":" + std::to_string(pc.misses) +
                       ",\"cached_tokens\":" + std::to_string(pc.cached_tokens) +
                       ",\"entries\":" + std::to_string(pc.entries) +
                       ",\"bytes\":" + std::to_string(pc.bytes) +
                       ",\"evictions\":" + std::to_string(pc.evictions) +
                       ",\"last_cached_tokens\":" + std::to_string(pc.last_cached_tokens) +
                       ",\"last_prefill_tokens\":" + std::to_string(pc.last_prefill_tokens) +
                       ",\"last_reason\":\"" + esc(pc.last_reason) + "\"}";
  std::string json =
      route + diag + prefix + ",\"loaded_experts\":" + std::to_string(c.loads) +
      ",\"cache_hits\":" + std::to_string(c.hits) +
      ",\"cache_misses\":" + std::to_string(c.misses) +
      ",\"cache_evictions\":" + std::to_string(c.evictions) +
      ",\"memory\":{\"resident_peak_bytes\":" + std::to_string(m.resident_peak_bytes) +
      ",\"footprint_bytes\":" + std::to_string(m.footprint_bytes) +
      ",\"expert_cache_heap_bytes\":" + std::to_string(c.heap_bytes) +
      ",\"file_backed_bytes\":0},"
      "\"tier\":\"" + esc(h->tier) + "\"}";
  return json_out(json, buf, buf_len, written);
}

E0Status e0_turn_reset(E0Engine *h) {
  if (!h) return E0_ERR_INVALID;
  std::lock_guard<std::mutex> g(h->mu);
  if (h->active) {
    h->active->seal();
    h->active = nullptr;
  }
  sweep_terminal("turn_reset");
  if (h->net35 && h->st35) {
    h->net35->reset_state(*h->st35);
    h->last_recs.clear();
    h->has_recs = false;
  }
  if (h->net8b && h->st8b) {
    h->net8b->reset_state(*h->st8b);
    h->last_recs.clear();
    h->has_recs = false;
  }
  h->last_error.clear();
  return E0_OK;
}

E0Status e0_last_error(E0Engine *h, char *buf, size_t buf_len, size_t *written) {
  std::string msg;
  if (h) {
    std::lock_guard<std::mutex> g(h->mu);
    msg = h->last_error.empty()
              ? std::string("{\"error\":\"E-INTERNAL\",\"message\":\"no error recorded\"}")
              : h->last_error;
  } else {
    std::lock_guard<std::mutex> g(g_tok_err_mu);
    msg = g_create_error.empty()
              ? std::string("{\"error\":\"E-INTERNAL\",\"message\":\"no error recorded\"}")
              : g_create_error;
  }
  return json_out(msg, buf, buf_len, written);
}

void e0_engine_destroy(E0Engine *h) {
  if (!h) return;
  {
    std::lock_guard<std::mutex> pg(g_pool_mu);
    for (auto it = g_streams.begin(); it != g_streams.end();) {
      if (it->second && it->second->owner_of(h))
        it = g_streams.erase(it);
      else
        ++it;
    }
  }
  pool_free(g_engines, h);
}

extern "C++" {
namespace {

void tok_err(const std::string &msg) {
  std::lock_guard<std::mutex> g(g_tok_err_mu);
  g_create_error = msg;
}

template <class F>
bool etok_out(F f, std::string &out, const char *fail_what) {
  size_t w = 0;
  if (f(nullptr, 0, &w) != 0) {
    tok_err(etok_err_json());
    return false;
  }
  out.assign(w + 1, '\0');
  if (w && f(out.data(), w, &w) != 0) {
    tok_err(etok_err_json());
    return false;
  }
  out.resize(w);
  return true;
}

bool etok_ids(void *raw, const std::string &text, std::vector<int32_t> &ids) {
  size_t n = 0;
  if (etok_tokenize(raw, text.c_str(), nullptr, 0, &n) != 0) {
    tok_err(etok_err_json());
    return false;
  }
  ids.assign(n, 0);
  if (n && etok_tokenize(raw, text.c_str(), ids.data(), n, &n) != 0) {
    tok_err(etok_err_json());
    return false;
  }
  return true;
}

}  // namespace
}  // extern "C++"

E0Tokenizer *e0_tok_create(const char *model_dir) {
  if (!is_dir(model_dir)) {
    tok_err(std::string("{\"error\":\"E-MODEL-PATH\",\"message\":\"directory does not exist: ") +
            esc(model_dir ? model_dir : "") + "\"}");
    return nullptr;
  }
  void *raw = etok_create(model_dir);
  if (!raw) {
    tok_err(etok_err_json());
    return nullptr;
  }
  return pool_new(g_toks, new E0Tokenizer(raw));
}

E0Status e0_tok_tokenize(E0Tokenizer *t, const char *text_json, char *buf, size_t buf_len,
                         size_t *written) {
  if (!t) return E0_ERR_INVALID;
  nlohmann::json v = nlohmann::json::parse(text_json ? text_json : "", nullptr, false);
  if (v.is_discarded() || !v.is_object()) {
    tok_err("{\"error\":\"E-INVALID-ARG\",\"message\":\"invalid text_json\"}");
    return E0_ERR_INTERNAL;
  }
  std::vector<std::vector<int32_t>> batches;
  if (v.contains("text") && v["text"].is_string()) {
    std::vector<int32_t> ids;
    if (!etok_ids(t->raw.get(), v["text"].get<std::string>(), ids)) return E0_ERR_INTERNAL;
    batches.push_back(std::move(ids));
  } else if (v.contains("texts") && v["texts"].is_array()) {
    for (const auto &item : v["texts"]) {
      if (!item.is_string()) {
        tok_err("{\"error\":\"E-INVALID-ARG\",\"message\":\"texts contains a non-string element\"}");
        return E0_ERR_INTERNAL;
      }
      std::vector<int32_t> ids;
      if (!etok_ids(t->raw.get(), item.get<std::string>(), ids)) return E0_ERR_INTERNAL;
      batches.push_back(std::move(ids));
    }
  } else {
    tok_err("{\"error\":\"E-INVALID-ARG\",\"message\":\"need text or texts\"}");
    return E0_ERR_INTERNAL;
  }
  nlohmann::json out = {{"tokens", batches}};
  return json_out(out.dump(), buf, buf_len, written);
}

E0Status e0_tok_apply_template(E0Tokenizer *t, const char *messages_json, char *buf,
                               size_t buf_len, size_t *written) {
  if (!t) return E0_ERR_INVALID;
  nlohmann::json v = nlohmann::json::parse(messages_json ? messages_json : "", nullptr, false);
  if (v.is_discarded()) {
    tok_err("{\"error\":\"E-INVALID-ARG\",\"message\":\"invalid messages_json\"}");
    return E0_ERR_INTERNAL;
  }
  nlohmann::json ctx = nlohmann::json::object();
  bool agp = true;
  if (v.is_array()) {
    ctx["messages"] = v;
  } else {
    for (auto it = v.begin(); it != v.end(); ++it) {
      if (it.key() == "add_generation_prompt" && it.value().is_boolean())
        agp = it.value().get<bool>();
      else
        ctx[it.key()] = it.value();
    }
    if (!ctx.contains("messages")) ctx["messages"] = nlohmann::json::array();
  }
  ctx["add_generation_prompt"] = agp;
  std::string cj = ctx.dump();
  void *raw = t->raw.get();
  std::string prompt;
  if (!etok_out([&](char *b, size_t l, size_t *w) {
        return etok_render(raw, cj.c_str(), b, l, w);
      },
      prompt, "render"))
    return E0_ERR_INTERNAL;
  std::vector<int32_t> ids;
  if (!etok_ids(raw, prompt, ids)) return E0_ERR_INTERNAL;
  nlohmann::json out = {{"prompt", prompt}, {"token_count", ids.size()}};
  return json_out(out.dump(), buf, buf_len, written);
}

E0Status e0_tok_decode(E0Tokenizer *t, const char *tokens_json, char *buf, size_t buf_len,
                       size_t *written) {
  if (!t) return E0_ERR_INVALID;
  nlohmann::json v = nlohmann::json::parse(tokens_json ? tokens_json : "", nullptr, false);
  if (v.is_discarded() || !v.is_object() || !v.contains("tokens") || !v["tokens"].is_array()) {
    tok_err("{\"error\":\"E-INVALID-ARG\",\"message\":\"need {\\\"tokens\\\":[...]} \"}");
    return E0_ERR_INTERNAL;
  }
  std::vector<int32_t> ids;
  for (const auto &x : v["tokens"]) {
    if (!x.is_number_integer()) {
      tok_err("{\"error\":\"E-INVALID-ARG\",\"message\":\"tokens contains a non-integer\"}");
      return E0_ERR_INTERNAL;
    }
    ids.push_back(x.get<int32_t>());
  }
  void *raw = t->raw.get();
  std::string text;
  if (!etok_out([&](char *b, size_t l, size_t *w) {
        return etok_decode(raw, ids.data(), ids.size(), b, l, w);
      },
      text, "decode"))
    return E0_ERR_INTERNAL;
  nlohmann::json out = {{"text", text}};
  return json_out(out.dump(), buf, buf_len, written);
}

void e0_tok_destroy(E0Tokenizer *t) { pool_free(g_toks, t); }

E0Status e0_test_forward(E0Engine *h, const int32_t *tokens, uint32_t n_tokens,
                         uint32_t steps, char *buf, size_t buf_len, size_t *written) {
  if (!h) return E0_ERR_INVALID;
  std::lock_guard<std::mutex> g(h->mu);
  if ((!h->net35 || !h->st35) && (!h->net8b || !h->st8b)) {
    h->last_error = "{\"error\":\"E-INVALID-ARG\",\"message\":\"forward context not enabled (config {\"forward\":\"35b\"|\"8b\"})\"}";
    return E0_ERR_INVALID;
  }
  std::vector<int32_t> tok(tokens, tokens + n_tokens);
  if (h->net35) return test_forward_run(h->net35, h->st35, h, tok, steps, buf, buf_len, written);
  return test_forward_run(h->net8b, h->st8b, h, tok, steps, buf, buf_len, written);
}

}  // extern "C"