Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// metal/src/engine/stream.cpp - see stream.h.
#include "stream.h"
#include "ref_quant.h"
#include "arena_budget.h"
#include <sys/sysctl.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
namespace strata::metal_engine {
namespace {
constexpr int NL = 48, NE = 512;
constexpr uint64_t PAGE = 16384;
double now_ms() { return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
int env_int(const char* n, int d) {
const char* e = std::getenv(n);
return e && *e ? std::atoi(e) : d;
}
} // namespace
bool ExpertStream::wanted(const gguf::File& f, const mtl::Context& cx) {
const char* e = std::getenv("STRATA_METAL_STREAM");
if (e && std::string(e) == "1") return true;
if (e && std::string(e) == "0") return false;
// auto: stream when the mapped weights (experts and dense) and ~12 GiB beside them (the prompt buffers, KV, the
// draft layer, macOS) do not fit the resident footprint cap: the GPU working set, or on a big Mac the RAM less the
// 30% macOS and its file cache keep (arena_budget.h: footprint_cap). M5 Max (115.4 GB working set, 96.2 GB cap):
// GSQ packs, IQ3_S and UD-Q4_K_XL stay resident, UD-Q5/Q6_K_XL stream; M4 32 GB (26.8 GB): every pack streams.
uint64_t all = 0;
for (const auto& kv : f.tensors)
if (kv.first != "per_layer_token_embd.weight") all += kv.second.bytes;
uint64_t phys = 0;
size_t len = sizeof phys;
sysctlbyname("hw.memsize", &phys, &len, nullptr, 0);
return !resident_fits((double) phys, (double) cx.recommended_working_set(), (double) all, 12.0 * (1ull << 30));
}
ExpertStream::~ExpertStream() {
{
std::lock_guard<std::mutex> g(mu_);
stop_ = true;
}
cv_.notify_all();
for (auto& t : io_) t.join();
if (fd_ >= 0) ::close(fd_);
}
bool ExpertStream::open(const gguf::File& f, std::string& err) {
fd_ = ::open(f.path.c_str(), O_RDONLY);
if (fd_ < 0) { err = "metal stream: cannot open " + f.path; return false; }
fcntl(fd_, F_NOCACHE, 1);
fcntl(fd_, F_RDAHEAD, 0);
lay_.resize(NL);
for (int l = 0; l < NL; ++l) {
const std::string b = "blk." + std::to_string(l) + ".";
const gguf::Tensor* g = f.find(b + "ffn_gate_exps.weight");
const gguf::Tensor* u = f.find(b + "ffn_up_exps.weight");
const gguf::Tensor* d = f.find(b + "ffn_down_exps.weight");
if (!g || !u || !d) { err = "metal stream: layer " + std::to_string(l) + " has no routed experts"; return false; }
lay_[(size_t) l] = {g->offset, g->bytes / NE, u->offset, u->bytes / NE, d->offset, d->bytes / NE, -1};
}
// the expert-contiguous copy beside the GGUF, when it is this GGUF's (sizes in its header) and its first and
// last experts match the GGUF's bytes
if (env_int("STRATA_METAL_CONTIG", 1) != 0) {
const std::string cp = f.path + ".experts";
const int cfd = ::open(cp.c_str(), O_RDONLY);
bool ok = cfd >= 0;
std::vector<uint64_t> off(NL);
if (ok) {
uint8_t h[64];
std::vector<uint64_t> t((size_t) NL * 4);
struct stat st {};
::fstat(fd_, &st);
ok = ::pread(cfd, h, 64, 0) == 64 && std::memcmp(h, "STRX", 4) == 0 &&
::pread(cfd, t.data(), t.size() * 8, 64) == (ssize_t) (t.size() * 8);
uint32_t ver = 0, nl = 0, ne = 0;
uint64_t src = 0;
std::memcpy(&ver, h + 4, 4); std::memcpy(&nl, h + 8, 4); std::memcpy(&ne, h + 12, 4); std::memcpy(&src, h + 16, 8);
ok = ok && ver == 1 && nl == (uint32_t) NL && ne == (uint32_t) NE && src == (uint64_t) st.st_size;
for (int l = 0; ok && l < NL; ++l) {
const Lay& x = lay_[(size_t) l];
ok = t[(size_t) l * 4 + 1] == x.gs && t[(size_t) l * 4 + 2] == x.us && t[(size_t) l * 4 + 3] == x.ds;
off[(size_t) l] = t[(size_t) l * 4];
}
if (ok) { // spot check: layer 0's first and layer 47's last expert, every slice
for (const int l : {0, NL - 1}) {
const Lay& x = lay_[(size_t) l];
const int e = l == 0 ? 0 : NE - 1;
std::vector<uint8_t> a(x.gs + x.us + x.ds), b(a.size());
ok = ok && ::pread(cfd, a.data(), a.size(), (off_t) (off[(size_t) l] + (uint64_t) e * a.size())) == (ssize_t) a.size() &&
::pread(fd_, b.data(), x.gs, (off_t) (x.g + (uint64_t) e * x.gs)) == (ssize_t) x.gs &&
::pread(fd_, b.data() + x.gs, x.us, (off_t) (x.u + (uint64_t) e * x.us)) == (ssize_t) x.us &&
::pread(fd_, b.data() + x.gs + x.us, x.ds, (off_t) (x.d + (uint64_t) e * x.ds)) == (ssize_t) x.ds &&
a == b;
}
}
}
if (ok) {
fcntl(cfd, F_NOCACHE, 1);
fcntl(cfd, F_RDAHEAD, 0);
cfd_ = cfd;
coff_ = off;
std::fprintf(stderr, "strata metal stream: experts read from %s (one request per expert)\n", cp.c_str());
} else if (cfd >= 0) {
::close(cfd);
std::fprintf(stderr, "strata metal stream: %s is not this GGUF's (sizes or bytes differ): the GGUF is read\n",
cp.c_str());
}
}
// size classes: one LRU pool per class. The layers' slot sizes (9 in the IQ3_XXS mix) are merged into a few
// classes, each as big as its largest member, so that no member wastes more than STRATA_METAL_CLASS_WASTE (default
// 0.15) of its slot: every class must hold a whole layer's experts for a prompt chunk, so fewer classes = less
// memory tied up in those minimums.
{
const char* ew = std::getenv("STRATA_METAL_CLASS_WASTE");
const double waste = ew ? std::atof(ew) : 0.15;
std::vector<uint64_t> pages_of(NL);
std::vector<uint64_t> sizes;
for (int l = 0; l < NL; ++l) {
const Lay& x = lay_[(size_t) l];
pages_of[(size_t) l] = (x.gs + x.us + x.ds + PAGE - 1) / PAGE;
sizes.push_back(pages_of[(size_t) l]);
}
std::sort(sizes.begin(), sizes.end());
sizes.erase(std::unique(sizes.begin(), sizes.end()), sizes.end());
std::vector<uint64_t> tops; // each class's slot pages, largest first
for (auto it = sizes.rbegin(); it != sizes.rend(); ++it)
if (tops.empty() || (double) *it < (1.0 - waste) * (double) tops.back()) tops.push_back(*it);
for (uint64_t t : tops) { Cls c; c.pages = t; cls_.push_back(c); }
for (int l = 0; l < NL; ++l) {
int best = 0; // the smallest class that holds the layer's slot
for (int c = 0; c < (int) cls_.size(); ++c)
if (cls_[(size_t) c].pages >= pages_of[(size_t) l]) best = c;
lay_[(size_t) l].cls = best;
}
}
nio_ = std::max(1, env_int("STRATA_METAL_IO_THREADS", 6));
split_ = std::max(1, std::min(8, env_int("STRATA_METAL_READ_SPLIT", 1)));
// The read-ahead's depth and where its experts wait in the LRU, measured on the M4 (notes-mac-ssd.md, 3 interleaved
// pairs each): Q2_0 (1.38 MB experts) top 10 at the LRU front 12.2 tok/s vs top 6 at the tail 11.3; the IQ3_XXS
// mix (1.75 MB, the SSD is the limit) top 6 at the tail 8.06 vs top 10 at the front 7.43.
uint64_t eb = 0;
for (const auto& x : lay_) eb += x.gs + x.us + x.ds;
const bool big = (double) eb / NL > 1.5e6;
pre_k_ = env_int("STRATA_METAL_PREFETCH", big ? 6 : 10);
pre_k2_ = env_int("STRATA_METAL_PREFETCH2", 0);
prefetch_ = pre_k_ > 0;
pre_mru_ = env_int("STRATA_METAL_PREFETCH_MRU", big ? 0 : 1) != 0;
// timing only (WRONG answers): every missing expert is pointed at slot 0, nothing is read
fake_ = env_int("STRATA_METAL_STREAM_FAKE", 0) != 0;
return true;
}
bool ExpertStream::read(void* dst, uint64_t len, uint64_t off) const {
uint8_t* p = (uint8_t*) dst;
while (len > 0) {
const ssize_t r = ::pread(fd_, p, (size_t) std::min<uint64_t>(len, 64ull << 20), (off_t) off);
if (r <= 0) return false;
p += r;
off += (uint64_t) r;
len -= (uint64_t) r;
}
return true;
}
bool ExpertStream::load_dense(gguf::File& f, mtl::Context& cx, const std::vector<std::string>& skip, mtl::Buffer& out,
std::string& err) {
const double t0 = now_ms();
std::vector<gguf::Tensor*> ts;
uint64_t tot = 0;
for (auto& kv : f.tensors) {
gguf::Tensor& t = kv.second;
if (t.name.find("_exps") != std::string::npos) continue;
// the n-gram table is read by the host from the SSD, a few rows at a time (src/ngram/ple_reader.cpp). It is its own shard in
// the ISTA / Unsloth files; in agentionai's Gyro files it is the first 28.8 GB of the one file, and must not be read in here
if (t.name == "per_layer_token_embd.weight") continue;
if (std::find(skip.begin(), skip.end(), t.name) != skip.end()) { t.offset = 0; continue; }
ts.push_back(&t);
tot += (t.bytes + 255) / 256 * 256;
}
out = cx.alloc((tot + PAGE - 1) / PAGE * PAGE);
std::vector<uint64_t> src(ts.size()), dst(ts.size());
uint64_t o = 0;
for (size_t i = 0; i < ts.size(); ++i) {
src[i] = ts[i]->offset;
dst[i] = o;
o += (ts[i]->bytes + 255) / 256 * 256;
}
std::atomic<size_t> next{0};
std::atomic<bool> ok{true};
std::vector<std::thread> th;
for (int k = 0; k < 4; ++k)
th.emplace_back([&] {
for (;;) {
const size_t i = next++;
if (i >= ts.size()) break;
if (!read(out.as<uint8_t>() + dst[i], ts[i]->bytes, src[i])) ok = false;
}
});
for (auto& x : th) x.join();
if (!ok) { err = "metal stream: reading the dense tensors failed"; return false; }
for (size_t i = 0; i < ts.size(); ++i) ts[i]->offset = dst[i];
std::fprintf(stderr, "strata metal stream: %zu dense tensors read into memory (%.2f GB, %.1f s; %zu left out)\n", ts.size(),
tot / 1e9, (now_ms() - t0) / 1e3, skip.size());
return true;
}
int ExpertStream::cls_of(int s) const { // the class ranges are contiguous
int ci = 0;
while (!(s >= cls_[(size_t) ci].base && s < cls_[(size_t) ci].base + cls_[(size_t) ci].n)) ++ci;
return ci;
}
void ExpertStream::lru_unlink(int s) {
Slot& x = slots_[(size_t) s];
Cls& k = cls_[(size_t) cls_of(s)];
if (x.prev >= 0) slots_[(size_t) x.prev].next = x.next; else k.head = x.next;
if (x.next >= 0) slots_[(size_t) x.next].prev = x.prev; else k.tail = x.prev;
x.prev = x.next = -1;
}
void ExpertStream::lru_back(int s) {
lru_unlink(s);
Cls& k = cls_[(size_t) cls_of(s)];
Slot& x = slots_[(size_t) s];
x.prev = k.tail;
x.next = -1;
if (k.tail >= 0) slots_[(size_t) k.tail].next = s;
k.tail = s;
if (k.head < 0) k.head = s;
}
void ExpertStream::lru_front(int s) {
lru_unlink(s);
Cls& k = cls_[(size_t) cls_of(s)];
Slot& x = slots_[(size_t) s];
x.next = k.head;
x.prev = -1;
if (k.head >= 0) slots_[(size_t) k.head].prev = s;
k.head = s;
if (k.tail < 0) k.tail = s;
}
int ExpertStream::victim(int ci) {
Cls& k = cls_[(size_t) ci];
for (int s = k.tail; s >= 0; s = slots_[(size_t) s].prev)
if (slots_[(size_t) s].state.load() == 0 && slots_[(size_t) s].pin != stamp_) {
Slot& x = slots_[(size_t) s];
if (x.layer >= 0) {
const int key = x.layer * NE + x.e;
where_[(size_t) key] = -1;
smap_.as<uint32_t>()[key] = 0xFFFFFFFFu;
}
x.layer = x.e = -1;
return s;
}
return -1;
}
// One piece of an expert: slice part / split_ (gate, up, down), piece part % split_ of it. The pieces of one expert
// go to several I/O threads, so a single miss is read with several requests in flight (its latency, not only the
// SSD's throughput, is what a layer waits for).
void ExpertStream::read_part(int s, int l, int e, int part) {
const Lay& x = lay_[(size_t) l];
if (cfd_ >= 0) { // the contiguous copy: the whole expert, piece part of split_ (the slot keeps gate | up | down)
const uint64_t len = x.gs + x.us + x.ds, src = coff_[(size_t) l] + (uint64_t) e * len;
const uint64_t a = len * (uint64_t) part / (uint64_t) split_ / 4096 * 4096;
const uint64_t b = part + 1 == split_ ? len : len * (uint64_t) (part + 1) / (uint64_t) split_ / 4096 * 4096;
uint8_t* p = host_[(size_t) s] + a;
uint64_t n = b - a, o = src + a;
while (n > 0) {
const ssize_t r = ::pread(cfd_, p, (size_t) n, (off_t) o);
if (r <= 0) {
std::fprintf(stderr, "strata metal stream: reading expert %d of layer %d failed\n", e, l);
std::exit(6);
}
p += r; o += (uint64_t) r; n -= (uint64_t) r;
}
bytes_read_ += b - a;
return;
}
const int k = part / split_, p = part % split_;
const uint64_t len = k == 0 ? x.gs : (k == 1 ? x.us : x.ds);
const uint64_t src = (k == 0 ? x.g : (k == 1 ? x.u : x.d)) + (uint64_t) e * len;
const uint64_t dst = k == 0 ? 0 : (k == 1 ? x.gs : x.gs + x.us);
const uint64_t a = len * (uint64_t) p / (uint64_t) split_ / 4096 * 4096;
const uint64_t b = p + 1 == split_ ? len : len * (uint64_t) (p + 1) / (uint64_t) split_ / 4096 * 4096;
if (b > a && !read(host_[(size_t) s] + dst + a, b - a, src + a)) {
std::fprintf(stderr, "strata metal stream: reading expert %d of layer %d failed\n", e, l);
std::exit(6);
}
bytes_read_ += b - a;
}
void ExpertStream::io_main() {
for (;;) {
Job j;
{
std::unique_lock<std::mutex> g(mu_);
cv_.wait(g, [&] { return stop_ || !urgent_q_.empty() || !pre_q_.empty(); });
if (stop_) return;
if (!urgent_q_.empty()) { j = urgent_q_.front(); urgent_q_.pop_front(); }
else { j = pre_q_.front(); pre_q_.pop_front(); }
slots_[(size_t) j.slot].state = 2;
}
read_part(j.slot, j.layer, j.e, j.part);
bool done = false;
{
std::lock_guard<std::mutex> g(mu_);
Slot& x = slots_[(size_t) j.slot];
if (--x.left == 0) {
smap_.as<uint32_t>()[j.layer * NE + j.e] = code_[(size_t) j.slot];
x.state = 0;
--outstanding_;
++reads_;
done = true;
}
}
if (done) done_cv_.notify_all();
}
}
void ExpertStream::submit(int s, int l, int e, bool urgent) {
Slot& x = slots_[(size_t) s];
x.layer = (int16_t) l;
x.e = (int16_t) e;
x.state = 1;
x.pre = !urgent;
const int parts = cfd_ >= 0 ? split_ : 3 * split_;
x.left = parts;
++outstanding_;
where_[(size_t) (l * NE + e)] = s;
for (int p = 0; p < parts; ++p) (urgent ? urgent_q_ : pre_q_).push_back({s, l, e, p});
}
// a queued read-ahead nobody wants: its pieces dropped, the slot freed (only before any piece has started)
void ExpertStream::cancel_slot(int s) {
Slot& x = slots_[(size_t) s];
for (auto it = pre_q_.begin(); it != pre_q_.end();) it = it->slot == s ? pre_q_.erase(it) : it + 1;
where_[(size_t) (x.layer * NE + x.e)] = -1;
x.layer = x.e = -1;
x.state = 0;
x.pre = false;
x.left = 0;
--outstanding_;
++pre_cancel_;
lru_back(s);
}
// a read-ahead that is wanted now: its queued pieces go first
void ExpertStream::promote_slot(int s) {
std::vector<Job> mine;
for (auto it = pre_q_.begin(); it != pre_q_.end();)
if (it->slot == s) { mine.push_back(*it); it = pre_q_.erase(it); }
else ++it;
for (auto it = mine.rbegin(); it != mine.rend(); ++it) urgent_q_.push_front(*it);
}
bool ExpertStream::make_arena(mtl::Context& cx, uint64_t budget, const std::string& profile, int min_slots, std::string& err,
const std::function<uint64_t()>& recheck, const std::function<bool(uint64_t)>& grow) {
const double t0 = now_ms();
shrink_to_ = 0;
for (auto& k : cls_) k.n = 0; // a second try (after a shrink) lays the arena out again
// the ranking: the profile's pairs, then every pair it misses, layer-interleaved
std::vector<std::pair<int, int>> rank;
std::vector<uint8_t> seen((size_t) NL * NE, 0);
if (FILE* f = std::fopen(profile.c_str(), "rb")) {
char magic[4];
uint32_t h[5];
if (std::fread(magic, 1, 4, f) == 4 && std::memcmp(magic, "STRP", 4) == 0 && std::fread(h, 4, 5, f) == 5 &&
h[1] == (uint32_t) NL && h[2] == (uint32_t) NE) {
for (uint32_t i = 0; i < h[4]; ++i) {
uint16_t p[2];
if (std::fread(p, 2, 2, f) != 2) break;
if (p[0] < NL && p[1] < NE && !seen[(size_t) p[0] * NE + p[1]]) {
seen[(size_t) p[0] * NE + p[1]] = 1;
rank.push_back({p[0], p[1]});
}
}
}
std::fclose(f);
} else {
std::fprintf(stderr, "strata metal stream: no expert profile at %s: the arena starts layer-interleaved\n", profile.c_str());
}
const size_t from_profile = rank.size();
for (int e = 0; e < NE; ++e)
for (int l = 0; l < NL; ++l)
if (!seen[(size_t) l * NE + e]) rank.push_back({l, e});
// the pool of each size class: at least min_slots (a prompt chunk's layer needs up to 512 at once), then the
// top-ranked pairs that fit in the budget
uint64_t used = 0;
for (const auto& k : cls_) used += (uint64_t) min_slots * k.pages * PAGE;
std::vector<std::pair<int, int>> fill;
for (const auto& p : rank) {
Cls& k = cls_[(size_t) lay_[(size_t) p.first].cls];
const uint64_t cost = k.n < min_slots ? 0 : k.pages * PAGE;
if (used + cost > budget) break;
used += cost;
++k.n;
fill.push_back(p);
}
int base = 0;
for (auto& k : cls_) {
k.n = std::max(k.n, min_slots);
k.base = base;
base += k.n;
}
// the slots placed in chunks of at most maxBufferLength (and 8 GB), up to 4 (the kernels' buffers 0, 7, 8, 9)
// (8 GB each unless the arena needs bigger ones to fit 4: the M4 refused one 20.6 GB buffer, the M5 Max takes an
// 89 GB arena for UD-Q5_K_XL in 4 of ~23 GB; STRATA_METAL_ARENA_CHUNK_GB sets it)
uint64_t total = 0, biggest = 0;
for (const auto& k : cls_) {
total += (uint64_t) k.n * k.pages * PAGE;
biggest = std::max<uint64_t>(biggest, k.pages * PAGE);
}
uint64_t want = std::max<uint64_t>(8ull << 30, total / 4 + 4 * biggest);
if (const char* e = std::getenv("STRATA_METAL_ARENA_CHUNK_GB")) want = (uint64_t) (std::atof(e) * (1ull << 30));
const uint64_t cmax = std::min<uint64_t>(cx.max_buffer_length(), want) / PAGE * PAGE;
code_.assign((size_t) base, 0);
host_.assign((size_t) base, nullptr);
std::vector<uint64_t> csize;
std::vector<std::pair<int, uint64_t>> at((size_t) base);
uint64_t cur = 0;
for (const auto& k : cls_)
for (int s = k.base; s < k.base + k.n; ++s) {
if (csize.empty() || cur + k.pages * PAGE > cmax) {
if (!csize.empty()) csize.back() = cur;
csize.push_back(0);
cur = 0;
}
at[(size_t) s] = {(int) csize.size() - 1, cur / PAGE};
cur += k.pages * PAGE;
}
if (!csize.empty()) csize.back() = cur;
if (csize.size() > 4) { err = "metal stream: the arena needs more than 4 buffers (lower STRATA_METAL_ARENA_GB)"; return false; }
if (recheck) { // the memory macOS can give, asked again just before the arena is committed
uint64_t total = 0;
for (uint64_t b : csize) total += b;
const uint64_t allowed = recheck();
if (allowed + (64ull << 20) < total) {
shrink_to_ = allowed;
err = "metal stream: less memory than planned is free now: the arena shrinks";
return false;
}
}
for (uint64_t b : csize) { // anonymous memory wrapped no-copy: a prompt's buffers can alias a region of it
void* m = mmap(nullptr, (size_t) b, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
if (m == MAP_FAILED) { err = "metal stream: cannot map the arena"; return false; }
chunks_.push_back(cx.wrap(m, (size_t) b));
arena_bytes_ += b;
}
for (int s = 0; s < base; ++s) {
code_[(size_t) s] = ((uint32_t) at[(size_t) s].first << 28) | (uint32_t) at[(size_t) s].second;
host_[(size_t) s] = chunks_[(size_t) at[(size_t) s].first].as<uint8_t>() + at[(size_t) s].second * PAGE;
}
smap_ = cx.alloc((size_t) NL * NE * 4);
std::memset(smap_.ptr, 0xFF, (size_t) NL * NE * 4);
slots_ = std::vector<Slot>((size_t) base);
retired_.assign((size_t) base, 0);
where_.assign((size_t) NL * NE, -1);
for (auto& k : cls_) {
for (int s = k.base; s < k.base + k.n; ++s) {
slots_[(size_t) s].prev = s == k.base ? -1 : s - 1;
slots_[(size_t) s].next = s + 1 == k.base + k.n ? -1 : s + 1;
}
k.head = k.n ? k.base : -1;
k.tail = k.n ? k.base + k.n - 1 : -1;
}
for (int i = 0; i < nio_; ++i) io_.emplace_back([this] { io_main(); });
// the fill: the ranked pairs in steps (best first; each step in file order) on the I/O threads; before each step
// grow() asks whether macOS still has the memory for it (arena_grow); the best-ranked ends at the LRU front
uint64_t step = 1536ull << 20;
if (const char* e = std::getenv("STRATA_METAL_ARENA_STEP_GB")) step = std::max<uint64_t>(64ull << 20, (uint64_t) (std::atof(e) * 1e9));
std::vector<int> next_free(cls_.size());
for (size_t c = 0; c < cls_.size(); ++c) next_free[c] = cls_[c].base;
size_t done = 0;
bool stopped = false;
uint64_t filled = 0;
while (done < fill.size()) {
size_t j = done;
uint64_t sb = 0;
while (j < fill.size() && (sb < step || j == done)) sb += cls_[(size_t) lay_[(size_t) fill[j++].first].cls].pages * PAGE;
if (grow && !grow(sb)) { stopped = true; break; }
std::vector<std::pair<int, int>> order(fill.begin() + (long) done, fill.begin() + (long) j);
std::sort(order.begin(), order.end(), [&](const auto& a, const auto& b) {
return lay_[(size_t) a.first].g + (uint64_t) a.second * lay_[(size_t) a.first].gs <
lay_[(size_t) b.first].g + (uint64_t) b.second * lay_[(size_t) b.first].gs;
});
{
std::lock_guard<std::mutex> g(mu_);
for (const auto& p : order) {
const int c = lay_[(size_t) p.first].cls;
submit(next_free[(size_t) c]++, p.first, p.second, true);
}
}
cv_.notify_all();
{
std::unique_lock<std::mutex> g(mu_);
done_cv_.wait(g, [&] { return outstanding_ == 0; });
}
filled += sb;
done = j;
}
uint64_t retired_bytes = 0;
size_t nretired = 0;
{
std::unique_lock<std::mutex> g(mu_);
if (stopped) // the slots not filled stay unused (their pages never touched), all but min_slots of each class
for (size_t c = 0; c < cls_.size(); ++c) {
const Cls& k = cls_[c];
for (int s = std::max(next_free[c], k.base + std::min(min_slots, k.n)); s < k.base + k.n; ++s) {
lru_unlink(s);
retired_[(size_t) s] = 1;
retired_bytes += k.pages * PAGE;
++nretired;
}
}
// the LRU order = the rank order (best at the front)
for (size_t i = done; i-- > 0;) lru_front(where_[(size_t) (fill[i].first * NE + fill[i].second)]);
}
if (stopped)
std::fprintf(stderr, "strata metal stream: the arena's fill stopped after %.2f GB: macOS's free memory reached what it "
"keeps; %.2f GB of the arena (%zu slots) stay unused\n", filled / 1e9, retired_bytes / 1e9, nretired);
arena_live_ = arena_bytes_ - retired_bytes;
fill_s_ = (now_ms() - t0) / 1e3;
fill_bytes_ = bytes_read_.load();
uint64_t pairs = 0;
for (const auto& k : cls_) pairs += (uint64_t) k.n;
std::fprintf(stderr, "strata metal stream: expert arena %.2f GB, %llu slots in %zu size classes, %zu experts read from the "
"profile's top (%zu ranked) in %.1f s (%.2f GB/s); %d I/O threads, read-ahead top %d; %zu buffers\n",
(arena_bytes_ - retired_bytes) / 1e9, (unsigned long long) (pairs - nretired), cls_.size(), done, from_profile, fill_s_,
fill_bytes_ / 1e9 / std::max(1e-9, fill_s_), nio_, prefetch_ ? pre_k_ : 0, chunks_.size());
// Wire the filled arena (mlock). Unwired, the arena is ordinary anonymous memory whenever the GPU is not running a command buffer
// (the GPU's own wiring ends a few seconds after the last one; a Metal residency set did not hold it either, measured), and
// macOS compresses it when anything else wants memory: a 20 GB file written beside the idle engine took tens of GB of the arena
// into the compressor in seconds and froze the 128 GB M5 Max (watchdog panic, 2026-10-13). Wired, a file write can only
// shrink the file cache. STRATA_METAL_WIRE_ARENA=0/1; the default is on for a Mac of 64 GB or more (the M4 32 GB was measured
// unwired). Not when the fill stopped early for lack of memory (the unused slots would be wired too).
{
uint64_t phys = 0;
size_t plen = sizeof phys;
sysctlbyname("hw.memsize", &phys, &plen, nullptr, 0);
const char* we = std::getenv("STRATA_METAL_WIRE_ARENA");
const bool want = we && *we ? std::atoi(we) != 0 : (double) phys >= kBigMac;
if (want && !stopped) {
const double tp = now_ms();
uint64_t pinned = 0;
int failed = 0;
for (const auto& c : chunks_) {
if (mlock(c.ptr, c.bytes) == 0) pinned += c.bytes;
else ++failed;
}
std::fprintf(stderr, "strata metal stream: %.1f GB of the arena wired (mlock) in %.0f ms%s; macOS cannot compress it while "
"the engine idles (STRATA_METAL_WIRE_ARENA=0 turns it off)\n", pinned / 1e9, now_ms() - tp,
failed ? "; SOME CHUNKS COULD NOT BE WIRED" : "");
}
}
(void) err;
return true;
}
// the experts a predicted routing names, most likely first (by the best logit over the window's tokens)
void ExpertStream::ahead(int ln, int T, const float* pred, int K) {
K = std::min(K, 32);
std::pair<float, int> pw[8 * 32];
int np = 0;
for (int t = 0; t < T; ++t) {
const float* lg = pred + (size_t) t * NE;
int top[32];
int nt = 0;
for (int e = 0; e < NE; ++e) {
if (nt < K) { top[nt++] = e; continue; }
int m = 0;
for (int j = 1; j < K; ++j) if (lg[top[j]] < lg[top[m]]) m = j;
if (lg[e] > lg[top[m]]) top[m] = e;
}
for (int j = 0; j < nt; ++j) {
int q = 0;
while (q < np && pw[q].second != top[j]) ++q;
if (q == np) pw[np++] = {lg[top[j]], top[j]};
else pw[q].first = std::max(pw[q].first, lg[top[j]]);
}
}
std::sort(pw, pw + np, [](const auto& a, const auto& b) { return a.first > b.first; });
bool any = false;
{
std::lock_guard<std::mutex> g(mu_);
for (int j = 0; j < np; ++j) {
const int key = ln * NE + pw[j].second;
const int s0 = where_[(size_t) key];
if (s0 >= 0) { lru_front(s0); continue; }
const int s = victim(lay_[(size_t) ln].cls);
if (s < 0) break;
submit(s, ln, pw[j].second, false);
slots_[(size_t) s].pin = stamp_;
if (pre_mru_) lru_front(s);
else lru_back(s);
++pre_issued_;
any = true;
}
}
if (any) cv_.notify_all();
}
void ExpertStream::at_layer(int l, int T, int topk, const int32_t* ids, const float* pred, const float* pred2) {
const double t0 = now_ms();
const bool prompt = T > 8; // a prompt chunk (the GEMM path): nearly every expert of the layer is used
if (l == 0) { if (prompt) ++p_chunks_; else ++windows_; }
++stamp_; // the slots this layer uses are pinned until the next handshake
bool used[NE] = {};
int want[NE];
int nw = 0;
for (int i = 0; i < T * topk; ++i) {
const int e = ids[i];
if (e < 0 || e >= NE || used[e]) continue;
used[e] = true;
want[nw++] = e;
}
if (fake_) {
for (int j = 0; j < nw; ++j)
if (where_[(size_t) (l * NE + want[j])] < 0) smap_.as<uint32_t>()[l * NE + want[j]] = code_[0];
host_ms_ += now_ms() - t0;
return;
}
std::vector<int> wait;
{
std::lock_guard<std::mutex> g(mu_);
// read-aheads of this layer (or older) that were not started and are not wanted: dropped (the slot goes free)
{
std::vector<int> drop;
for (const Job& j : pre_q_) {
if (j.layer > l || slots_[(size_t) j.slot].state.load() != 1) continue;
if (!(j.layer == l && used[j.e]) && std::find(drop.begin(), drop.end(), j.slot) == drop.end()) drop.push_back(j.slot);
}
for (int s : drop) cancel_slot(s);
}
for (int j = 0; j < nw; ++j) // pass 1: pin what is resident (or on its way), so no miss below evicts it
if (where_[(size_t) (l * NE + want[j])] >= 0) slots_[(size_t) where_[(size_t) (l * NE + want[j])]].pin = stamp_;
for (int j = 0; j < nw; ++j) {
const int e = want[j], key = l * NE + e;
uint64_t& lk = prompt ? p_lookups_ : lookups_;
++lk;
int s = where_[(size_t) key];
if (s >= 0) {
Slot& x = slots_[(size_t) s];
x.pin = stamp_;
const int st = x.state.load();
if (st == 0) { if (x.pre) ++(prompt ? p_pre_ : pre_hits_); else ++(prompt ? p_hits_ : hits_); }
else {
++(prompt ? p_pre_ : pre_hits_);
if (!prompt) ++late_;
promote_slot(s); // a queued read-ahead becomes urgent
wait.push_back(s);
}
if (!(prompt && x.pre)) lru_front(s); // a prompt's streamed experts stay at the LRU tail
x.pre = false;
continue;
}
++(prompt ? p_misses_ : misses_);
s = victim(lay_[(size_t) l].cls);
if (s < 0) { std::fprintf(stderr, "strata metal stream: no free slot (layer %d)\n", l); std::exit(6); }
submit(s, l, e, true);
slots_[(size_t) s].pin = stamp_;
if (!prompt) lru_front(s);
wait.push_back(s);
}
}
if (!wait.empty()) {
cv_.notify_all();
const double tw = now_ms();
std::unique_lock<std::mutex> g(mu_);
done_cv_.wait(g, [&] {
for (int s : wait) if (slots_[(size_t) s].state.load() != 0) return false;
return true;
});
(prompt ? p_wait_ms_ : wait_ms_) += now_ms() - tw;
}
if (prompt) {
// a chunk uses nearly all of the next layer's experts: every one not in the arena is streamed in behind this
// layer's compute, into the LRU tail (the decode-hot experts stay)
if (l + 1 < NL) {
bool any = false;
std::lock_guard<std::mutex> g(mu_);
const int ln = l + 1;
for (int e = 0; e < NE; ++e) {
if (where_[(size_t) (ln * NE + e)] >= 0) continue;
const int s = victim(lay_[(size_t) ln].cls);
if (s < 0) break;
submit(s, ln, e, false);
slots_[(size_t) s].pin = stamp_;
++p_streamed_;
any = true;
}
if (any) cv_.notify_all();
}
(prompt ? p_host_ms_ : host_ms_) += now_ms() - t0;
return;
}
// read-ahead: layer l + 1 (each token's top pre_k_ of the predicted router logits), then layer l + 2 (top pre_k2_)
if (prefetch_ && pred && l + 1 < NL) ahead(l + 1, T, pred, pre_k_);
if (pre_k2_ > 0 && pred2 && l + 2 < NL) ahead(l + 2, T, pred2, pre_k2_);
host_ms_ += now_ms() - t0;
}
// A region of `bytes` at the end of one arena buffer, for a prompt's scratch buffers: its slots are emptied and held
// (state 3) while a prompt runs (hold_region), and are ordinary slots again for decoding.
uint8_t* ExpertStream::region(uint64_t bytes) {
for (int c = (int) chunks_.size() - 1; c >= 0; --c) {
if (chunks_[(size_t) c].bytes < bytes + (64ull << 20)) continue;
uint8_t* end = chunks_[(size_t) c].as<uint8_t>() + chunks_[(size_t) c].bytes;
uint8_t* lo = end - (bytes + PAGE - 1) / PAGE * PAGE;
region_slots_.clear();
uint8_t* first = end;
for (int s = 0; s < (int) slots_.size(); ++s) {
const uint8_t* h = host_[(size_t) s];
const uint64_t sz = cls_[(size_t) cls_of(s)].pages * PAGE;
if (h + sz > lo && h < end) { region_slots_.push_back(s); first = std::min(first, (uint8_t*) h); }
}
{ // a class left with fewer slots than a prompt chunk's layer can use (NE) would stop with "no free slot": a small
// arena (a Mac with little free memory at start) gets the prompt's buffers as their own allocation instead
std::vector<int> stolen(cls_.size(), 0);
for (int s : region_slots_) ++stolen[(size_t) cls_of(s)];
bool tight = false;
for (size_t i = 0; i < cls_.size(); ++i) tight = tight || (cls_[i].n > 0 && cls_[i].n - stolen[i] < NE);
if (tight) return standalone_region(bytes);
}
region_ = lo;
region_bytes_ = (uint64_t) (end - lo);
std::fprintf(stderr, "strata metal stream: %.2f GB of the arena (%zu slots) hold a prompt's buffers while it runs\n",
bytes / 1e9, region_slots_.size());
return lo;
}
// no chunk is big enough for the prompt's buffers (an arena that shrank at the start, a 24 GB Mac): the fallback
// inside the loop above is only reached when a chunk fits, so the buffers get their own allocation here too
return standalone_region(bytes);
}
// The prompt's buffers as their own anonymous allocation (fresh zero pages), when no region of the arena's slots
// takes them: no slots are held while a prompt runs, and the arena keeps decoding with all of its slots.
uint8_t* ExpertStream::standalone_region(uint64_t bytes) {
const uint64_t sz = (bytes + PAGE - 1) / PAGE * PAGE;
void* m = mmap(nullptr, (size_t) sz, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
if (m == MAP_FAILED) return nullptr;
region_slots_.clear();
region_ = (uint8_t*) m;
region_bytes_ = 0; // fresh zero pages; nothing in the arena to empty
std::fprintf(stderr, "strata metal stream: the arena is small: the prompt's %.2f GB of buffers are their own "
"allocation, not a region of its slots\n", bytes / 1e9);
return region_;
}
void ExpertStream::hold_region(bool hold) {
std::unique_lock<std::mutex> g(mu_);
if (hold) {
// nothing may be reading into those slots: the queued read-aheads are dropped, the running ones waited for
std::vector<int> drop;
for (const Job& j : pre_q_)
if (slots_[(size_t) j.slot].state.load() == 1 && std::find(drop.begin(), drop.end(), j.slot) == drop.end())
drop.push_back(j.slot);
for (int s : drop) cancel_slot(s);
done_cv_.wait(g, [&] { return outstanding_ == 0; });
for (int s : region_slots_) {
Slot& x = slots_[(size_t) s];
if (x.layer >= 0) {
const int key = x.layer * NE + x.e;
where_[(size_t) key] = -1;
smap_.as<uint32_t>()[key] = 0xFFFFFFFFu;
}
x.layer = x.e = -1;
x.pre = false;
x.state = 3;
}
if (region_) std::memset(region_, 0, (size_t) region_bytes_);
} else {
for (int s : region_slots_) {
slots_[(size_t) s].state = 0;
if (!retired_[(size_t) s]) lru_back(s); // a retired slot's memory serves the prompt only
}
}
}
void ExpertStream::reset_stats() {
lookups_ = hits_ = pre_hits_ = late_ = misses_ = pre_issued_ = pre_cancel_ = windows_ = 0;
p_lookups_ = p_hits_ = p_pre_ = p_misses_ = p_streamed_ = p_chunks_ = 0;
p_wait_ms_ = p_host_ms_ = 0;
bytes_read_ = 0;
reads_ = 0;
wait_ms_ = host_ms_ = 0;
}
void ExpertStream::report(FILE* f) const {
if (!ready()) return;
if (p_chunks_)
std::fprintf(f, "expert stream prompts %llu chunks: %llu distinct lookups (%.1f per layer), in the arena %.4f, streamed "
"ahead %.4f, missed %.4f; %llu streamed in; host %.1f ms per chunk, of it waiting %.1f ms\n",
(unsigned long long) p_chunks_, (unsigned long long) p_lookups_, p_lookups_ / 48.0 / p_chunks_,
p_hits_ / std::max(1.0, (double) p_lookups_), p_pre_ / std::max(1.0, (double) p_lookups_),
p_misses_ / std::max(1.0, (double) p_lookups_), (unsigned long long) p_streamed_, p_host_ms_ / p_chunks_,
p_wait_ms_ / p_chunks_);
const double w = std::max<uint64_t>(1, windows_);
std::fprintf(f,
"expert stream %llu windows: %llu distinct lookups (%.1f per window), hits %.4f, read-ahead hits "
"%.4f (%llu of them late), misses %.4f (%.1f per window); read-aheads %llu (%.1f per window, %llu dropped); %.1f MB "
"read per window (%llu reads); host %.2f ms per window, of it waiting %.2f ms\n",
(unsigned long long) windows_, (unsigned long long) lookups_, lookups_ / w,
hits_ / std::max(1.0, (double) lookups_), pre_hits_ / std::max(1.0, (double) lookups_),
(unsigned long long) late_, misses_ / std::max(1.0, (double) lookups_), misses_ / w,
(unsigned long long) pre_issued_, pre_issued_ / w, (unsigned long long) pre_cancel_,
(bytes_read_.load() - 0) / 1e6 / w, (unsigned long long) reads_.load(), host_ms_ / w, wait_ms_ / w);
}
} // namespace strata::metal_engine