Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
// src/kernels/cpu/pool.cpp - P2.S3: the CPU expert pool. Read pool.hpp first; it explains the protocol.
#include "strata/kernels/cpu/pool.hpp"
#include "strata/core/progress.hpp"
#include "strata/platform/aux_cpus.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
#include "strata/kernels/cpu/numa_pool.hpp"
#include <algorithm>
#include <atomic>
#include <cmath>
#include <chrono>
#include "strata/platform/cpu_relax.hpp"
#if defined(_MSC_VER)
#include <intrin.h> // __cpuidex, __rdtsc (STRATA_POOL_UMWAIT)
#endif
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <stdexcept>
#include <string>
#include <thread>
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include "pool_affinity_win.hpp"
#else
#include <pthread.h>
#include <sched.h>
#include <unistd.h>
#include "pool_affinity_linux.hpp"
#endif
namespace strata::kernels::cpu {
namespace {
// STRATA_POOL_UMWAIT=<TSC cycles> (experimental, x86 WAITPKG only): a parked worker waits with UMONITOR/UMWAIT on the
// epoch's cache line instead of spinning on _mm_pause. It wakes when the host publishes (a write to that line) or
// after the cycles, in the light C0.1 state; on a laptop the CPU then gives back the power the spin took, which the
// GPU's share of the budget gets. 0 or unset (or no WAITPKG): the spin, unchanged. STRATA_POOL_UMWAIT_C02=1: the
// deeper C0.2 state (slower wake). MSVC x64 only for now (the intrinsics need no target flag there); other builds spin.
uint64_t umwait_cycles() {
static const uint64_t c = [] {
const char* e = std::getenv("STRATA_POOL_UMWAIT");
if (e == nullptr || std::atoll(e) <= 0) return (uint64_t) 0;
#if defined(_MSC_VER) && defined(_M_X64) && !defined(__clang__) // MSVC proper: clang-cl (the Windows HIP build) has no WAITPKG intrinsics by default
int r[4] = {0, 0, 0, 0};
__cpuidex(r, 7, 0);
if (((r[2] >> 5) & 1) == 0) {
std::fprintf(stderr, "strata: STRATA_POOL_UMWAIT: this CPU has no WAITPKG, the workers spin\n");
return (uint64_t) 0;
}
std::fprintf(stderr, "strata: STRATA_POOL_UMWAIT: parked workers wait with UMWAIT (%lld cycles)\n", std::atoll(e));
return (uint64_t) std::atoll(e);
#else
return (uint64_t) 0;
#endif
}();
return c;
}
unsigned umwait_state() {
static const unsigned s = [] { const char* e = std::getenv("STRATA_POOL_UMWAIT_C02"); return e && *e == '1' ? 0u : 1u; }();
return s; // 1 = C0.1 (light), 0 = C0.2
}
// STRATA_POOL_UMWAIT_ESC=<TSC cycles> (experimental): an escalating wait - C0.1 for the first cycles of a park, C0.2 after
// (a wait that has lasted that long is a GPU phase, a draft or a prompt read, where a slower wake costs little).
uint64_t umwait_escalate() {
static const uint64_t c = [] { const char* e = std::getenv("STRATA_POOL_UMWAIT_ESC"); return e ? (uint64_t) (std::max)(0LL, std::atoll(e)) : (uint64_t) 0; }();
return c;
}
constexpr uint64_t pack_head(uint32_t epoch, uint32_t n, uint32_t i) {
return ((uint64_t) epoch << 32) | ((uint64_t) n << 16) | (uint64_t) i;
}
} // namespace
namespace {
std::atomic<int> g_host_core{(int) HostCore::First};
} // namespace
void set_host_core(HostCore where) { g_host_core.store((int) where); }
HostCore host_core_setting() { return (HostCore) g_host_core.load(); }
namespace {
std::vector<int> g_pool_cores; // --pool-cores: set once at start-up, before the pool exists
} // namespace
void set_pool_cores(const std::vector<int>& cpus) { g_pool_cores = cpus; }
const std::vector<int>& pool_cores_setting() { return g_pool_cores; }
std::vector<LogicalCpuInfo> logical_cpu_layout() {
std::vector<LogicalCpuInfo> out;
#if defined(_WIN32)
DWORD len = 0;
GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len);
if (len == 0) return out;
std::vector<char> buf(len);
if (!GetLogicalProcessorInformationEx(RelationProcessorCore, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf.data(), &len))
return out;
// a `start /affinity` or a job limit on a PC with one processor group: only those processors are the process's
DWORD_PTR proc_mask = 0, sys_mask = 0;
const bool limited = GetActiveProcessorGroupCount() == 1 && GetProcessAffinityMask(GetCurrentProcess(), &proc_mask, &sys_mask);
long core = 0;
for (const char* p = buf.data(); p < buf.data() + len; p += ((const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*) p)->Size) {
const auto* e = (const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*) p;
if (e->Relationship != RelationProcessorCore) continue;
for (WORD group = 0; group < e->Processor.GroupCount; ++group) {
const GROUP_AFFINITY& g = e->Processor.GroupMask[group];
for (int bit = 0; bit < 64; ++bit)
if ((g.Mask & (KAFFINITY(1) << bit)) && (!limited || (proc_mask & (DWORD_PTR(1) << bit))))
out.push_back({(int) (g.Group * 64 + bit), core});
}
++core;
}
#elif defined(__linux__)
std::vector<unsigned long> mask;
std::vector<int> allowed;
if (detail::get_thread_affinity(mask, &allowed) != 0) return out;
auto read = [](int cpu, const char* what) -> long {
char path[96];
std::snprintf(path, sizeof path, "/sys/devices/system/cpu/cpu%d/topology/%s", cpu, what);
long v = -1;
if (std::FILE* f = std::fopen(path, "r")) {
if (std::fscanf(f, "%ld", &v) != 1) v = -1;
std::fclose(f);
}
return v;
};
for (int cpu : allowed) {
const long pkg = read(cpu, "physical_package_id"), core = read(cpu, "core_id");
out.push_back({cpu, pkg >= 0 && core >= 0 ? pkg * 1000000 + core : -1});
}
#else
for (unsigned i = 0; i < std::thread::hardware_concurrency(); ++i) out.push_back({(int) i, -1});
#endif
return out;
}
static CpuTopology detect_cpu_topology_impl(bool skip_first, PoolAffinity affinity);
// The fields the default worker count needs (pool_plan.hpp): the logical processors the process may use, the CPU
// package of the host core and of every worker core, whether the OS told physical cores from SMT siblings, the
// container CPU quota. On a multi-socket CPU (not hybrid) the host package comes first in `worker_cores`.
static void fill_plan_fields(CpuTopology& topo, bool skip_first) {
topo.cpu_quota = container_cpu_quota();
topo.worker_package.assign(topo.worker_cores.size(), -1);
#if defined(_WIN32)
(void) skip_first;
topo.logical = (int) GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
return; // (the sockets of a Windows box are not told apart: one package)
#elif defined(__APPLE__)
(void) skip_first;
topo.logical = (int) std::thread::hardware_concurrency();
return; // no SMT on Apple silicon and no sysfs: the cores are the processors
#else
std::vector<unsigned long> mask;
std::vector<int> allowed;
topo.logical = detail::get_thread_affinity(mask, &allowed) == 0 ? (int) allowed.size()
: (int) std::thread::hardware_concurrency();
auto read = [](int cpu, const char* what) -> long {
char path[96];
std::snprintf(path, sizeof path, "/sys/devices/system/cpu/cpu%d/topology/%s", cpu, what);
long v = -1;
if (std::FILE* f = std::fopen(path, "r")) {
if (std::fscanf(f, "%ld", &v) != 1) v = -1;
std::fclose(f);
}
return v;
};
bool known = !topo.worker_cores.empty() || topo.host_core >= 0;
auto pkg_of = [&](int cpu) {
const long pkg = read(cpu, "physical_package_id"), core = read(cpu, "core_id");
if (pkg < 0 || core < 0) known = false;
return (int) pkg;
};
if (topo.host_core >= 0) topo.host_package = pkg_of(topo.host_core);
for (size_t i = 0; i < topo.worker_cores.size(); ++i) topo.worker_package[i] = pkg_of(topo.worker_cores[i]);
topo.layout_known = known;
if (!known) return;
if (skip_first && !topo.is_hybrid && topo.host_core >= 0 && topo.host_package >= 0) {
// the host socket first (stable inside each socket): the default workers are the local ones
std::vector<size_t> order(topo.worker_cores.size());
for (size_t i = 0; i < order.size(); ++i) order[i] = i;
std::stable_partition(order.begin(), order.end(),
[&](size_t i) { return topo.worker_package[i] == topo.host_package; });
std::vector<int> cores, pkgs;
for (size_t i : order) { cores.push_back(topo.worker_cores[i]); pkgs.push_back(topo.worker_package[i]); }
topo.worker_cores = std::move(cores);
topo.worker_package = std::move(pkgs);
}
#endif
}
CpuTopology detect_cpu_topology(bool skip_first, PoolAffinity affinity) {
CpuTopology topo = detect_cpu_topology_impl(skip_first, affinity);
// --host-core last: the host takes the last core, the workers the others (the first included)
if (skip_first && host_core_setting() == HostCore::Last && !topo.is_hybrid && topo.host_core >= 0 &&
!topo.worker_cores.empty()) {
const int last = topo.worker_cores.back();
topo.worker_cores.pop_back();
topo.worker_cores.insert(topo.worker_cores.begin(), topo.host_core);
topo.host_core = last;
}
// --host-core sibling: the host on its core's other hardware thread; the first logical processor stays to the
// interrupts and to nothing else (a worker on it would share the host's core)
if (skip_first && host_core_setting() == HostCore::Sibling && topo.host_sibling >= 0) {
topo.worker_cores.erase(std::remove(topo.worker_cores.begin(), topo.worker_cores.end(), topo.host_sibling),
topo.worker_cores.end());
topo.host_core = topo.host_sibling;
}
fill_plan_fields(topo, skip_first);
return topo;
}
double container_cpu_quota() {
#if defined(__linux__)
// cgroup v2: "max 100000" (no limit) or "<quota> <period>"; v1: cpu.cfs_quota_us (-1: none) / cpu.cfs_period_us
if (std::FILE* f = std::fopen("/sys/fs/cgroup/cpu.max", "r")) {
char a[32] = {0};
long period = 0;
double q = 0;
if (std::fscanf(f, "%31s %ld", a, &period) == 2 && std::strcmp(a, "max") != 0 && period > 0)
q = std::atof(a) / (double) period;
std::fclose(f);
return q > 0 ? q : 0.0;
}
long quota = -1, period = 0;
if (std::FILE* f = std::fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r")) {
if (std::fscanf(f, "%ld", "a) != 1) quota = -1;
std::fclose(f);
}
if (std::FILE* f = std::fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r")) {
if (std::fscanf(f, "%ld", &period) != 1) period = 0;
std::fclose(f);
}
return quota > 0 && period > 0 ? (double) quota / (double) period : 0.0;
#else
return 0.0;
#endif
}
PoolPlanInput pool_plan_input(const CpuTopology& topo) {
PoolPlanInput in;
in.logical = topo.logical > 0 ? topo.logical : (int) std::thread::hardware_concurrency();
in.cpu_quota = topo.cpu_quota;
if (topo.is_hybrid) { in.p_cores = topo.p_cores; in.e_cores = topo.e_cores; }
if (!topo.layout_known) return in; // cores_per_package stays empty
// the cores per package, the host package first; the host core counts in its package
std::vector<std::pair<int, int>> counts; // (package, cores)
auto add = [&](int pkg) {
for (auto& c : counts)
if (c.first == pkg) { ++c.second; return; }
counts.emplace_back(pkg, 1);
};
if (topo.host_core >= 0) add(topo.host_package);
for (size_t i = 0; i < topo.worker_cores.size(); ++i)
add(i < topo.worker_package.size() ? topo.worker_package[i] : -1);
if (topo.host_core >= 0) { // (the host package first even when it has no worker core listed)
auto it = std::find_if(counts.begin(), counts.end(), [&](const auto& c) { return c.first == topo.host_package; });
if (it != counts.begin()) std::iter_swap(counts.begin(), it);
}
for (const auto& c : counts) in.cores_per_package.push_back(c.second);
return in;
}
int planned_host_core(PoolAffinity affinity) { return detect_cpu_topology(true, affinity).host_core; }
static CpuTopology detect_cpu_topology_impl(bool skip_first, PoolAffinity affinity) {
CpuTopology topo;
#if defined(_WIN32)
// Ask the OS rather than assuming a layout. `hardware_concurrency()` returns LOGICAL processors, and on
// every SMT machine half of them are siblings - pinning one worker to each of the first N would put two
// workers on each physical core and halve the bandwidth the expert kernel is bound by.
DWORD len = 0;
GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len);
if (len == 0) {
// Preserve group identity even when detailed core topology is unavailable.
const WORD groups = GetActiveProcessorGroupCount();
for (WORD group = 0; group < groups; ++group) {
const DWORD count = GetActiveProcessorCount(group);
if (count == 0 || count == (DWORD) -1 || count > 64) continue;
for (DWORD i = 0; i < count; ++i) topo.worker_cores.push_back((int) group * 64 + (int) i);
}
if (skip_first && !topo.worker_cores.empty()) {
topo.host_core = topo.worker_cores.front();
topo.worker_cores.erase(topo.worker_cores.begin());
}
// The OS did not say which logical processors share a core, and every one is listed as a core above: the default
// worker count must not count them as cores (30 workers on a 16c/32t PC would sit two to a core). With the layout
// unknown the plan takes half the logical processors (pool_plan.hpp), as on Linux without sysfs.
topo.layout_known = false;
return topo;
}
std::vector<char> buf(len);
if (GetLogicalProcessorInformationEx(RelationProcessorCore,
(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf.data(), &len)) {
const char* p = buf.data();
const char* end = p + len;
struct CoreDesc {
uint8_t efficiency = 0;
bool has_smt = false;
std::vector<int> lps;
};
std::vector<CoreDesc> descs;
while (p < end) {
const auto* e = (const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX*) p;
if (e->Relationship == RelationProcessorCore) {
CoreDesc cd;
cd.efficiency = e->Processor.EfficiencyClass;
cd.has_smt = (e->Processor.Flags & LTP_PC_SMT) != 0;
for (WORD group = 0; group < e->Processor.GroupCount; ++group) {
const GROUP_AFFINITY& g = e->Processor.GroupMask[group];
for (int bit = 0; bit < 64; ++bit) {
if (g.Mask & (KAFFINITY(1) << bit)) {
cd.lps.push_back((int) (g.Group * 64 + bit));
}
}
}
if (!cd.lps.empty()) {
descs.push_back(std::move(cd));
}
}
p += e->Size;
}
uint8_t min_eff = 255, max_eff = 0;
for (const auto& c : descs) {
min_eff = (std::min)(min_eff, c.efficiency);
max_eff = (std::max)(max_eff, c.efficiency);
}
topo.is_hybrid = (max_eff > min_eff);
if (topo.is_hybrid) {
for (const auto& c : descs) {
if (c.efficiency == max_eff) {
topo.p_cores++;
topo.p_threads += (int) c.lps.size();
} else {
topo.e_cores++;
}
}
} else {
topo.p_cores = (int) descs.size();
for (const auto& c : descs) topo.p_threads += (int) c.lps.size();
}
if (affinity == PoolAffinity::All || !topo.is_hybrid) {
// #642 (from Hardin22's fork): on a hybrid CPU the P-cores first (the host takes the first of them), so a
// pool smaller than the core count (setup's --pool-workers for a hybrid CPU) runs on the P-cores and the
// first E-cores rather than on whatever the OS numbered first. All cores alike: the order is unchanged.
if (topo.is_hybrid)
std::stable_sort(descs.begin(), descs.end(),
[](const CoreDesc& x, const CoreDesc& y) { return x.efficiency > y.efficiency; });
for (const auto& c : descs) topo.worker_cores.push_back(c.lps[0]);
if (skip_first && !topo.worker_cores.empty()) {
topo.host_core = topo.worker_cores.front();
topo.worker_cores.erase(topo.worker_cores.begin());
if (descs.front().lps.size() > 1) topo.host_sibling = descs.front().lps[1];
}
return topo;
}
// Hybrid CPU with Auto or PCores affinity:
// Prioritize Performance cores:
// 1. Primary logical processor of each P-core (avoids SMT resource contention)
// 2. SMT sibling logical processors of P-cores
// 3. E-cores (only as overflow in Auto mode)
std::vector<int> p_primaries;
std::vector<int> p_siblings;
std::vector<int> e_cores;
for (const auto& c : descs) {
if (c.efficiency == max_eff) {
p_primaries.push_back(c.lps[0]);
for (size_t s = 1; s < c.lps.size(); ++s) {
p_siblings.push_back(c.lps[s]);
}
} else {
for (int lp : c.lps) e_cores.push_back(lp);
}
}
if (skip_first && !p_primaries.empty()) {
topo.host_core = p_primaries.front();
p_primaries.erase(p_primaries.begin());
for (const auto& c : descs)
if (c.lps[0] == topo.host_core) {
if (c.lps.size() > 1) topo.host_sibling = c.lps[1];
break;
}
}
for (int cpu : p_primaries) topo.worker_cores.push_back(cpu);
for (int cpu : p_siblings) topo.worker_cores.push_back(cpu);
if (affinity != PoolAffinity::PCores) {
for (int cpu : e_cores) topo.worker_cores.push_back(cpu);
}
return topo;
}
#else
// The logical CPUs this process may run on, ONE PER PHYSICAL CORE (issue #40): SMT siblings share a core's
// load/store bandwidth, so a worker on each would put two workers on one core, as the Windows branch above
// explains. sysfs names each CPU's (package, core); the first allowed CPU of each pair is kept, so a taskset
// that leaves out the first sibling still gets its core. Without sysfs every allowed CPU counts, as before.
auto topo_read = [](int cpu, const char* what) -> long {
char path[96];
std::snprintf(path, sizeof path, "/sys/devices/system/cpu/cpu%d/topology/%s", cpu, what);
long v = -1;
if (std::FILE* f = std::fopen(path, "r")) {
if (std::fscanf(f, "%ld", &v) != 1) v = -1;
std::fclose(f);
}
return v;
};
auto cap_read = [](int cpu) -> long {
char path[96];
std::snprintf(path, sizeof path, "/sys/devices/system/cpu/cpu%d/cpu_capacity", cpu);
long v = -1;
if (std::FILE* f = std::fopen(path, "r")) {
if (std::fscanf(f, "%ld", &v) != 1) v = -1;
std::fclose(f);
}
return v;
};
// #798: a cpulist ("0-7,16") of one of the hybrid PMU's CPU sets: /sys/devices/cpu_core/cpus and cpu_atom/cpus
auto cpulist_read = [](const char* path, std::vector<int>& out) -> bool {
std::FILE* f = std::fopen(path, "r");
if (!f) return false;
char text[1024] = {0};
const size_t n = std::fread(text, 1, sizeof text - 1, f);
std::fclose(f);
text[n] = '\0';
for (const char* p = text; *p;) {
char* end = nullptr;
const long lo = std::strtol(p, &end, 10);
if (end == p) break;
long hi = lo;
if (*end == '-') {
p = end + 1;
hi = std::strtol(p, &end, 10);
}
for (long c = lo; c <= hi && c < 4096; ++c) out.push_back((int) c);
p = end;
while (*p == ',' || *p == ' ' || *p == '\n') ++p;
}
return !out.empty();
};
std::vector<int> allowed;
std::vector<unsigned long> allowed_mask;
if (detail::get_thread_affinity(allowed_mask, &allowed) != 0) {
for (unsigned i = 0; i < std::thread::hardware_concurrency(); ++i) allowed.push_back((int) i);
}
struct CoreLinux {
int cpu = -1;
long pkg = -1;
long core = -1;
long cap = -1;
bool is_sibling = false;
};
std::vector<CoreLinux> all_cpus;
std::vector<std::pair<long, long>> seen_phys;
long max_cap = 0, min_cap = 1000000;
for (int cpu : allowed) {
CoreLinux cl;
cl.cpu = cpu;
cl.pkg = topo_read(cpu, "physical_package_id");
cl.core = topo_read(cpu, "core_id");
cl.cap = cap_read(cpu);
if (cl.cap > 0) {
max_cap = (std::max)(max_cap, cl.cap);
min_cap = (std::min)(min_cap, cl.cap);
}
if (cl.pkg >= 0 && cl.core >= 0) {
const std::pair<long, long> key{cl.pkg, cl.core};
if (std::find(seen_phys.begin(), seen_phys.end(), key) != seen_phys.end()) {
cl.is_sibling = true;
} else {
seen_phys.push_back(key);
}
}
all_cpus.push_back(cl);
}
// #798: which CPUs are E-cores. Intel's hybrid PMU lists them (cpu_atom/cpus beside cpu_core/cpus); without it a
// CPU is an E-core when its capacity is under 90% of the largest. Turbo Boost Max 3.0 gives the "favored"
// P-cores a slightly higher capacity (1024 against 1012 on a Core Ultra 7 270K Plus): "capacity == the maximum"
// counted 2 of its 8 P-cores and started 1 pool worker.
// #3262: Meteor Lake and Arrow Lake-H list their LP-E cores (SoC tile, no L3, ~2.5 GHz) in a third set,
// /sys/devices/cpu_lowpower/cpus, that is in neither of the other two; they are E-cores too (and sort last).
std::vector<int> pmu_core, pmu_atom, pmu_lowpower;
const bool pmu = cpulist_read("/sys/devices/cpu_core/cpus", pmu_core) &&
cpulist_read("/sys/devices/cpu_atom/cpus", pmu_atom);
cpulist_read("/sys/devices/cpu_lowpower/cpus", pmu_lowpower);
auto is_e = [&](const CoreLinux& cl) {
return hybrid_cpu_is_efficiency(cl.cpu, cl.cap, max_cap, pmu, pmu_core, pmu_atom, pmu_lowpower);
};
bool any_e = false, any_p = false;
for (const auto& cl : all_cpus) (is_e(cl) ? any_e : any_p) = true;
topo.is_hybrid = any_e && any_p;
if (topo.is_hybrid) {
for (const auto& cl : all_cpus) {
if (!is_e(cl)) {
if (!cl.is_sibling) topo.p_cores++;
topo.p_threads++;
} else {
if (!cl.is_sibling) topo.e_cores++;
}
}
} else {
topo.p_cores = (int) seen_phys.size();
topo.p_threads = (int) all_cpus.size();
}
// the host core's SMT sibling, an allowed CPU of the same (package, core) (--host-core sibling)
auto sibling_of = [&](int cpu) {
for (const auto& h : all_cpus)
if (h.cpu == cpu && h.pkg >= 0 && h.core >= 0)
for (const auto& cl : all_cpus)
if (cl.cpu != cpu && cl.pkg == h.pkg && cl.core == h.core) return cl.cpu;
return -1;
};
if (affinity == PoolAffinity::All || !topo.is_hybrid) {
if (topo.is_hybrid) // #642: the P-cores first (see the Windows branch)
std::stable_sort(all_cpus.begin(), all_cpus.end(),
[&](const CoreLinux& x, const CoreLinux& y) {
return is_e(x) != is_e(y) ? !is_e(x) : x.cap > y.cap;
});
for (const auto& cl : all_cpus) {
if (!cl.is_sibling) topo.worker_cores.push_back(cl.cpu);
}
if (skip_first && !topo.worker_cores.empty()) {
topo.host_core = topo.worker_cores.front();
topo.worker_cores.erase(topo.worker_cores.begin());
topo.host_sibling = sibling_of(topo.host_core);
}
return topo;
}
// Hybrid CPU on Linux:
std::vector<int> p_primaries;
std::vector<int> p_siblings;
std::vector<int> e_cores;
for (const auto& cl : all_cpus) {
if (!is_e(cl)) {
if (!cl.is_sibling) p_primaries.push_back(cl.cpu);
else p_siblings.push_back(cl.cpu);
} else {
e_cores.push_back(cl.cpu);
}
}
if (skip_first && !p_primaries.empty()) {
topo.host_core = p_primaries.front();
p_primaries.erase(p_primaries.begin());
topo.host_sibling = sibling_of(topo.host_core);
}
for (int cpu : p_primaries) topo.worker_cores.push_back(cpu);
for (int cpu : p_siblings) topo.worker_cores.push_back(cpu);
if (affinity != PoolAffinity::PCores) {
for (int cpu : e_cores) topo.worker_cores.push_back(cpu);
}
return topo;
#endif
return topo;
}
std::vector<int> physical_cores(bool skip_first, PoolAffinity affinity) {
return detect_cpu_topology(skip_first, affinity).worker_cores;
}
namespace {
bool pin_this_thread(int core, [[maybe_unused]] int worker = -1) {
if (core < 0) return false;
#if defined(_WIN32)
return detail::set_thread_group_affinity(core, worker);
#else
const int error = detail::pin_thread_to_cpu(core);
if (error != 0)
std::fprintf(stderr, "strata cpu pool: affinity for worker %d (CPU %d) failed: %d; previous affinity kept\n",
worker, core, error);
return error == 0;
#endif
}
#if defined(__linux__)
// Where the NUMA topology is read from: /sys. STRATA_TEST_SYSFS_ROOT=<dir> reads <dir>/devices/system/{node,cpu} instead,
// so a test can give the pool a made-up two-node machine (pool_tasks_test); nobody sets it for real.
std::string sysfs_root() {
const char* e = std::getenv("STRATA_TEST_SYSFS_ROOT");
return e != nullptr && *e != '\0' ? std::string(e) : std::string("/sys");
}
// The NUMA node of a logical CPU (sysfs), or -1.
int cpu_numa_node(int cpu) {
if (cpu < 0) return -1;
const std::string root = sysfs_root();
char path[512];
for (int node = 0; node < 64; ++node) {
std::snprintf(path, sizeof path, "%s/devices/system/cpu/cpu%d/node%d", root.c_str(), cpu, node);
if (access(path, F_OK) == 0) return node;
}
return -1;
}
#endif
// The NUMA nodes with CPUs this machine has (0.2.2: the split is tried by itself from 2 on). Linux: the sysfs node folder
// (numa_pool.hpp); Windows: GetNumaHighestNodeNumber; any other system, or a sandbox that hides the folder: 1.
int numa_node_count() {
#if defined(__linux__)
return (std::max)(1, numa_cpu_node_count_at(sysfs_root() + "/devices/system/node"));
#elif defined(_WIN32)
ULONG highest = 0;
return GetNumaHighestNodeNumber(&highest) ? (int) highest + 1 : 1;
#else
return 1;
#endif
}
// The real-node split for threads on `thread_core` (numa_pool.hpp). Linux only: the placement of the experts' memory
// (first touch by threads pinned to a node, move_pages to check) has no Windows or macOS version.
NumaSplitDecision decide_real_numa_split([[maybe_unused]] const std::vector<int>& thread_core, [[maybe_unused]] int n_workers,
[[maybe_unused]] bool pinned) {
#if defined(__linux__)
// the expert source's own reading of it (expert_source.cpp): exactly "1"
const char* rotate = std::getenv("STRATA_EXCHANGE_ROTATE");
return decide_numa_split(thread_core, n_workers, pinned, rotate != nullptr && std::strcmp(rotate, "1") == 0, cpu_numa_node);
#else
NumaSplitDecision d;
d.why = "the split needs Linux";
return d;
#endif
}
} // namespace
ThreadAffinity pin_current_thread(int core) {
if (core < 0) return {};
#if defined(_WIN32)
ThreadAffinity previous;
ULONG target = 0;
if (!detail::get_thread_cpu_sets(previous.cpu_sets) || !detail::cpu_set_for_core(core, target) ||
!SetThreadSelectedCpuSets(GetCurrentThread(), &target, 1)) {
std::fprintf(stderr, "strata cpu pool: host CPU Set selection for processor %d failed: %lu; previous placement kept\n",
core, (unsigned long) GetLastError());
return {};
}
previous.valid = true;
return previous;
#else
ThreadAffinity previous;
int error = detail::get_thread_affinity(previous.mask);
if (error == 0) error = detail::pin_thread_to_cpu(core);
if (error != 0) {
std::fprintf(stderr, "strata cpu pool: host affinity for CPU %d failed: %d; previous affinity kept\n", core, error);
return {};
}
previous.valid = true;
return previous;
#endif
}
void restore_thread_affinity(const ThreadAffinity& previous) {
if (!previous.valid) return;
#if defined(_WIN32)
// Clearing an originally empty selection restores process-default/all-group eligibility without
// turning the caller's implicit Windows 11 affinity into an explicit single-group hard mask.
if (!SetThreadSelectedCpuSets(GetCurrentThread(), previous.cpu_sets.empty() ? nullptr : previous.cpu_sets.data(),
(ULONG) previous.cpu_sets.size()))
std::fprintf(stderr, "strata cpu pool: host CPU Set restoration failed: %lu\n",
(unsigned long) GetLastError());
#else
const int error = detail::set_thread_affinity(previous.mask);
if (error != 0)
std::fprintf(stderr, "strata cpu pool: host affinity restoration failed: %d\n", error);
#endif
}
namespace {
std::atomic<const ExpertPool*> g_diag_pool{nullptr};
void diag_active_pool(std::FILE* f) {
if (const ExpertPool* p = g_diag_pool.load()) p->diag(f);
}
int64_t now_ms() {
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch())
.count();
}
int64_t now_ns() {
return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch())
.count();
}
// CBA: one plain global, written by the constructor and destructor of the pool that owns it; fine while pools are
// built and destroyed on one thread (generate's measure pools, then the main one), an atomic owner if that changes.
NumaSplitPlan g_numa_plan;
const ExpertPool* g_numa_owner = nullptr;
} // namespace
const NumaSplitPlan& active_numa_split() { return g_numa_plan; }
void ExpertPool::diag(std::FILE* f) const {
const uint64_t h = head_.load();
uint32_t batch = (uint32_t) (h >> 32), claimed = (uint32_t) h & 0xffffu, total = (uint32_t) (h >> 16) & 0xffffu,
done = done_.load();
// CBA: this summed-slices branch is never run by a test (it needs a diag call mid-phase), it is a trivial sum, a mid-phase
// diag test if it ever misreports
if (numa_n_ > 0 && mode_ >= 5) { // a split native batch never writes head_/done_: the slices' sums instead
claimed = total = done = 0;
for (int k = 0; k < numa_n_; ++k) {
const uint64_t sh = nsync_[(size_t) k].head.load();
batch = (uint32_t) (sh >> 32); // every slice head carries the batch's epoch
claimed += (uint32_t) sh & 0xffffu;
total += (uint32_t) (sh >> 16) & 0xffffu;
done += nsync_[(size_t) k].done.load();
}
}
std::fprintf(f, " expert pool: epoch %u, batch epoch %u: %u of %u jobs claimed, %u done; %u of %d workers parked, "
"%u sleeping; mode %d\n", epoch_.load(), batch, claimed, total, done, parked_count(), n_,
sleepers_.load(), mode_);
if (numa_n_ > 0) std::fprintf(f, " expert pool NUMA slices:%s\n", numa_counters().c_str());
std::fprintf(f, " expert pool threads:");
for (int i = 0; i < n_; ++i) {
const int32_t s = wstate_[(size_t) i].load();
if (s == kParked) std::fprintf(f, " w%d=parked", i);
else if (s == kSleeping) std::fprintf(f, " w%d=sleeping", i);
else if (s == kBetween) std::fprintf(f, " w%d=draining", i);
else std::fprintf(f, " w%d=job%d", i, s);
}
const int32_t hs = hstate_.load();
const char* hn = hs == kIdle ? "idle" : hs == kWaitParked ? "waiting for the workers to park"
: hs == kWaitDone ? "waiting for the jobs to finish" : "running a job";
std::fprintf(f, "; host %s", hn);
if (hs >= 0) std::fprintf(f, " %d", hs);
std::fprintf(f, " for %lld ms\n", (long long) (now_ms() - hstate_ms_.load()));
}
ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity affinity, int tasks)
: tasks_(tasks), host_works_(host_works), affinity_(affinity), topo_(detect_cpu_topology(true, affinity)) {
if (tasks < 0 || tasks > kMaxTasks) throw std::invalid_argument("expert pool tasks must be in 0..4096");
pinned_ = pin;
layout_ = logical_cpu_layout(); // now, before anything pins the caller (Linux reads the calling thread's allowed CPUs)
if (const char* e = std::getenv("STRATA_POOL_SPIN_US")) // a test knob; see kSpinBeforeSleep
spin_before_sleep_ = std::chrono::microseconds((std::max)(0, std::atoi(e)));
// STRATA_NUMA_SPLIT, read here for the worker count (the split itself is decided below, once the count is known).
// 0.2.2: with 2 or more NUMA nodes and neither variable set the split is tried by itself, and STRATA_NUMA_POOL=0 turns that
// off (numa_pool.hpp). One node: pick_numa_pool says Unchanged and nothing below differs from the pool before the split.
const char* env = std::getenv("STRATA_NUMA_SPLIT");
const int numa_nodes = numa_node_count();
const NumaPoolPick pool_pick = pick_numa_pool(std::getenv("STRATA_NUMA_POOL"), env, numa_nodes);
if (!pool_pick.log.empty()) std::fprintf(stderr, "%s\n", pool_pick.log.c_str());
if (pool_pick.choice == NumaPoolChoice::Off) env = nullptr;
bool auto_pool = pool_pick.choice == NumaPoolChoice::Auto; // the split without STRATA_NUMA_SPLIT: only if it can run
bool env_off = !auto_pool && (env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0);
bool env_known = auto_pool || (!env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0));
const PoolPlanInput plan_in = pool_plan_input(topo_);
const PoolPlan wplan = plan_pool_workers(plan_in);
// --pool-cores (experimental, off by default): the person's own list of logical processors for the workers. The plan above
// was made from the real topology; only the cores the workers are pinned to (and, by default, their count) come from the list.
if (!pool_cores_setting().empty()) {
const PoolCoresCheck chk = check_pool_cores(pool_cores_setting(), layout_, topo_.host_core);
static std::atomic<bool> told{false}; // (a throw-away pool of --pool-affinity measure builds the same list again)
const bool say = !told.exchange(true);
if (say)
for (const std::string& w : chk.warnings) std::fprintf(stderr, "strata cpu pool: --pool-cores: %s\n", w.c_str());
if (chk.cpus.empty()) {
if (say)
std::fprintf(stderr, "strata cpu pool: --pool-cores: no usable logical processor in the list; the pool uses its "
"default placement\n");
} else {
topo_.worker_cores = chk.cpus;
topo_.worker_package.assign(chk.cpus.size(), -1);
explicit_cores_ = true;
}
}
// the split needs workers on both nodes: its cores are taken round-robin over the nodes (the packages when a core's node
// is unknown), so any count under the core count (the default or a configured one) spans them evenly (one node: the order
// is kept)
auto interleave_cores = [&] {
if (explicit_cores_) return; // --pool-cores: the person's order stays
if (topo_.worker_package.size() != topo_.worker_cores.size()) return;
std::vector<int> key = topo_.worker_package;
#if defined(__linux__)
std::vector<int> by_node;
for (int c : topo_.worker_cores) by_node.push_back(cpu_numa_node(c));
if (std::none_of(by_node.begin(), by_node.end(), [](int n) { return n < 0; })) key = by_node;
#endif
std::vector<int> cores, pkgs;
for (size_t i : interleave_packages(key)) {
cores.push_back(topo_.worker_cores[i]);
pkgs.push_back(topo_.worker_package[i]);
}
topo_.worker_cores = std::move(cores);
topo_.worker_package = std::move(pkgs);
};
// the worker count: configured, else the hybrid P cores, else the plan (`span`: every socket counts, the split's default)
auto pick_workers = [&](bool span, const char** limit) -> int {
if (n_workers > 0) return n_workers;
if (explicit_cores_) {
*limit = "your --pool-cores list";
return (int) topo_.worker_cores.size();
}
if (topo_.is_hybrid && affinity_ != PoolAffinity::All) return (std::max)(1, topo_.p_cores - 1);
if (span) {
PoolPlanInput span_in = plan_in;
span_in.span_packages = true;
const PoolPlan sp = plan_pool_workers(span_in);
*limit = sp.limit;
return (std::min)(sp.workers, (std::max)(1, (int) topo_.worker_cores.size()));
}
*limit = wplan.limit;
return (std::min)(wplan.workers, (std::max)(1, (int) topo_.worker_cores.size()));
};
if (auto_pool) {
// try the split on a copy: when it cannot run the cores, the count and the plan stay exactly what they were
const CpuTopology plain_topo = topo_;
interleave_cores();
const char* try_limit = nullptr;
const int try_n = pick_workers(true, &try_limit);
std::vector<int> thread_core;
for (int j = 0; j < try_n; ++j)
thread_core.push_back(pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1);
if (host_works_) thread_core.push_back(topo_.host_core);
const NumaSplitDecision tried = decide_real_numa_split(thread_core, try_n, pin);
if (!tried.on) {
topo_ = plain_topo;
auto_pool = env_known = false;
env_off = true;
std::fprintf(stderr, "strata cpu pool: NUMA pool off: %s (%d NUMA nodes found), the pool runs as before\n",
tried.why.c_str(), numa_nodes);
}
} else if (env_known) {
interleave_cores();
}
{
const char* limit = nullptr;
n_ = pick_workers(env_known, &limit);
if (limit != nullptr) plan_limit_ = limit;
}
plan_cores_ = wplan.cores;
plan_logical_ = plan_in.logical;
if (n_ < 1) n_ = 1;
scratch_.resize((size_t) n_);
wstate_.reset(new std::atomic<int32_t>[(size_t) n_]);
for (int i = 0; i < n_; ++i) wstate_[(size_t) i].store(kParked);
hstate_ms_.store(now_ms());
g_diag_pool.store(this);
strata::core::diag_pool_fn().store(&diag_active_pool);
split_.resize((size_t) kMaxSplit);
split_multi_.resize((size_t) kMaxSplitMulti);
// STRATA_NUMA_SPLIT: decided here, before any worker starts (a worker counts into its slice's `parked` from its
// first instruction), and never revised. Unset: the plan stays empty and nothing below runs.
NumaSplitPlan plan;
if (!env_off && !env_known) {
plan.why = std::string("unknown STRATA_NUMA_SPLIT value '") + env + "' (use 1 or test)";
std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());
} else if (env_known) {
plan.test = !auto_pool && std::strcmp(env, "test") == 0;
const int threads = n_ + (host_works_ ? 1 : 0); // thread j < n_: worker j; j == n_: the host
std::vector<int> slice((size_t) threads, 0);
std::string why;
if (threads < 2) {
why = "the pool has " + std::to_string(threads) + " thread, the split needs 2 or more";
} else if (plan.test) {
for (int j = 0; j < threads; ++j) slice[(size_t) j] = j % 2;
} else {
std::vector<int> thread_core;
for (int j = 0; j < threads; ++j)
thread_core.push_back(j == n_ ? topo_.host_core
: pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1);
const NumaSplitDecision d = decide_real_numa_split(thread_core, n_, pin);
why = d.why;
if (d.on) {
slice = d.slice;
plan.nodes[0] = d.nodes[0];
plan.nodes[1] = d.nodes[1];
for (int j = 0; j < threads; ++j) plan.cores[slice[(size_t) j]].push_back(thread_core[(size_t) j]);
}
}
plan.why = why;
plan.on = plan.why.empty();
if (!plan.on) { // a refusal after the slices were planned (unbalanced) leaves no half-filled plan behind
plan.nodes[0] = plan.nodes[1] = -1;
plan.cores[0].clear();
plan.cores[1].clear();
}
if (plan.on) {
numa_n_ = 2;
nsync_.reset(new NodeSync[2]);
wslice_.assign(slice.begin(), slice.begin() + n_);
hslice_ = host_works_ ? slice[(size_t) n_] : 0;
slice_threads_.assign(2, 0);
for (int k : slice) ++slice_threads_[(size_t) k];
if (auto_pool)
std::fprintf(stderr, "strata cpu pool: NUMA pool on: %d NUMA nodes found, one slice of the pool per node "
"(STRATA_NUMA_POOL=0 turns it off)\n", numa_nodes);
if (plan.test)
std::fprintf(stderr, "strata cpu pool: NUMA split on (test, no real nodes): slice 0 = %d threads, "
"slice 1 = %d threads\n", slice_threads_[0], slice_threads_[1]);
else
std::fprintf(stderr, "strata cpu pool: NUMA split on: slice 0 = node %d (%d threads), slice 1 = node %d "
"(%d threads)\n", plan.nodes[0], slice_threads_[0], plan.nodes[1], slice_threads_[1]);
} else {
std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());
}
}
if (n_workers > 0) {
// recommend, never force: a count above what this CPU uses well runs as configured, with a plain warning
// (STRATA_NUMA_SPLIT on: workers on the other socket are the point, that case is not warned about)
PoolPlanInput warn_in = plan_in;
warn_in.span_packages = plan.on;
static std::atomic<bool> warned{false};
const std::string why = pool_workers_warning(n_workers, warn_in, plan_pool_workers(warn_in));
if (!why.empty() && !warned.exchange(true)) std::fprintf(stderr, "strata cpu pool: warning: %s\n", why.c_str());
}
g_numa_plan = std::move(plan);
g_numa_owner = this;
threads_.reserve((size_t) n_);
for (int i = 0; i < n_; ++i) {
const int core = pin ? (i < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) i] : -1) : -1;
threads_.emplace_back([this, i, core] {
strata::platform::name_this_thread("strata-pool", i);
strata::aux_cpus::note_owned_thread(); // --aux-cpus never moves a worker
// a failed pin only costs locality under the split: the worker keeps the slice it was given
if (!pin_this_thread(core, i) && core >= 0 && numa_n_ > 0)
std::fprintf(stderr, "strata cpu pool: NUMA split: worker %d not pinned to CPU %d, its slice's rows may "
"be read remotely\n", i, core);
worker(i);
});
}
}
std::string ExpertPool::placement_line() const {
if (n_ <= 0) return "";
if (!pinned_) return "CPU workers: " + std::to_string(n_) + " workers, not pinned to logical processors (the OS places them)";
// a worker beyond the cores listed runs unpinned (more --pool-workers than --pool-cores entries)
std::vector<int> on(topo_.worker_cores.begin(), topo_.worker_cores.begin() + (std::min)((size_t) n_, topo_.worker_cores.size()));
std::string s = describe_worker_placement(on, topo_.host_core, layout_, explicit_cores_);
if ((int) on.size() < n_) s += " (" + std::to_string(n_ - (int) on.size()) + " more workers run unpinned)";
return s;
}
ExpertPool::~ExpertPool() {
const ExpertPool* self = this;
g_diag_pool.compare_exchange_strong(self, nullptr);
if (g_numa_owner == this) {
g_numa_plan = NumaSplitPlan{};
g_numa_owner = nullptr;
}
stop_.store(true, std::memory_order_release);
// Bump the epoch so a PARKED worker notices the stop flag rather than sleeping through it.
publish();
for (auto& t : threads_) t.join();
}
void ExpertPool::publish() {
// Both sides are seq_cst, and that is the whole lost-wakeup argument: a worker going to sleep does
// `sleepers_++` and then reads `epoch_`, the host does `epoch_++` and then reads `sleepers_`. In one total
// order at least one of them sees the other's write - the worker sees the new epoch and does not sleep, or
// the host sees the sleeper and notifies under the mutex the worker holds until it is inside `wait`.
// On x86 the fetch_add is a locked xadd either way, so this costs the token path nothing.
epoch_.fetch_add(1, std::memory_order_seq_cst);
if (sleepers_.load(std::memory_order_seq_cst) != 0) {
std::lock_guard<std::mutex> lk(sleep_mu_);
sleep_cv_.notify_all();
}
}
void ExpertPool::worker(int id) {
uint32_t seen = 0;
// ARRIVE at the park before the first wait, so `parked_ == n_` is true from construction. Counting only
// on the RETURN from a drain leaves `parked_` at 0 until each worker has finished one batch, and the first
// `run()` - which waits for `parked_ == n_` before publishing - then deadlocks. It deadlocks on the very
// first call, which is the good case; a version that deadlocked on the second would be far worse.
parked_of(id).fetch_add(1, std::memory_order_acq_rel);
for (;;) {
// Park: wait for work. `_mm_pause` rather than a bare spin because it yields the pipeline to the
// sibling hyperthread; `epoch_` is bumped once per LAYER, not once per expert, so most of these
// iterations are spent here with nothing to do.
//
// **AND NOTHING ELSE HAPPENS IN HERE.** This loop used to do `pauses_.fetch_add(1)` on every iteration
// - a locked read-modify-write, five workers against one cache line - so the workers spent their wait
// invalidating each other's caches and the very line the host writes to publish work. The counter was
// diagnostic and nothing branched on it. See the note on the atomics in pool.hpp.
//
// After `kSpinBeforeSleep` with no work the worker sleeps instead (issue #4). The clock is read once
// every 1024 pauses, so the spin itself is unchanged.
const auto parked_at = std::chrono::steady_clock::now();
uint32_t spins = 0;
const uint64_t um = umwait_cycles();
#if defined(_MSC_VER) && defined(_M_X64) && !defined(__clang__) // MSVC proper: clang-cl (the Windows HIP build) has no WAITPKG intrinsics by default
const uint64_t um_esc = umwait_escalate();
const uint64_t um_t0 = um != 0 && um_esc != 0 ? __rdtsc() : 0;
#endif
while (epoch_.load(std::memory_order_acquire) == seen) {
if (stop_.load(std::memory_order_relaxed)) return;
#if defined(_MSC_VER) && defined(_M_X64) && !defined(__clang__) // MSVC proper: clang-cl (the Windows HIP build) has no WAITPKG intrinsics by default
if (um != 0) { // STRATA_POOL_UMWAIT: wake on the publish's write to epoch_, or after `um` cycles
_umonitor((void*) &epoch_);
if (epoch_.load(std::memory_order_acquire) != seen) break;
const uint64_t now = __rdtsc();
const unsigned st = um_esc != 0 ? (now - um_t0 < um_esc ? 1u : 0u) : umwait_state();
_umwait(st, now + um);
if ((++spins & 15u) != 0) continue; // each wait is ~um cycles: look at the clock more often
} else
#endif
{
strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
}
if (std::chrono::steady_clock::now() - parked_at < spin_before_sleep_) continue;
std::unique_lock<std::mutex> lk(sleep_mu_);
wstate_[(size_t) id].store(kSleeping, std::memory_order_relaxed);
sleepers_.fetch_add(1, std::memory_order_seq_cst);
sleep_cv_.wait(lk, [&] {
return epoch_.load(std::memory_order_seq_cst) != seen || stop_.load(std::memory_order_relaxed);
});
sleepers_.fetch_sub(1, std::memory_order_relaxed);
wstate_[(size_t) id].store(kParked, std::memory_order_relaxed);
}
if (stop_.load(std::memory_order_acquire)) return;
// acquire: the batch this epoch published (`head`, and the description before it) is visible from here
seen = epoch_.load(std::memory_order_acquire);
parked_of(id).fetch_sub(1, std::memory_order_acq_rel); // leaving the park
// Drain: one claim per iteration, so a slow worker takes fewer experts and a fast one takes more.
// Every job is the same size (all experts are 1,382,400 bytes), so there is nothing to schedule. Only
// this epoch's jobs: if the host has already moved on, the claims fail and the worker parks again.
wstate_[(size_t) id].store(kBetween, std::memory_order_relaxed);
drain(id, scratch_[(size_t) id], seen);
wstate_[(size_t) id].store(kParked, std::memory_order_relaxed);
parked_of(id).fetch_add(1, std::memory_order_acq_rel); // back at the park
}
}
int ExpertPool::claim(uint32_t epoch) {
uint64_t h = head_.load(std::memory_order_acquire);
for (;;) {
if ((uint32_t) (h >> 32) != epoch) return -1; // not the batch this thread woke for
const uint32_t n = (uint32_t) (h >> 16) & 0xffffu, i = (uint32_t) h & 0xffffu;
if (i >= n) return -1; // exhausted
if (head_.compare_exchange_weak(h, h + 1, std::memory_order_acq_rel, std::memory_order_acquire))
return (int) i;
}
}
int ExpertPool::claim_slice(uint32_t epoch, int slice) {
NodeSync& ns = nsync_[(size_t) slice];
uint64_t h = ns.head.load(std::memory_order_acquire);
for (;;) {
if ((uint32_t) (h >> 32) != epoch) return -1; // not this batch, or not a mode-5/6 batch
const uint32_t n = (uint32_t) (h >> 16) & 0xffffu, i = (uint32_t) h & 0xffffu;
if (i >= n) return -1; // the slice is exhausted
if (ns.head.compare_exchange_weak(h, h + 1, std::memory_order_acq_rel, std::memory_order_acquire))
return ns.base + (int) i; // the description is stable from here
}
}
uint32_t ExpertPool::parked_count() const {
if (numa_n_ == 0) return parked_.load(std::memory_order_acquire);
uint32_t p = 0; // each slice's counter is bounded by its own workers, so the sum is n_ only when each is full
for (int k = 0; k < numa_n_; ++k) p += nsync_[(size_t) k].parked.load(std::memory_order_acquire);
return p;
}
std::string ExpertPool::numa_counters() const {
std::string s;
for (int k = 0; k < numa_n_; ++k) {
const NodeSync& ns = nsync_[(size_t) k];
const uint64_t h = ns.head.load();
char buf[200];
std::snprintf(buf, sizeof buf, "%s slice %d: batch epoch %u, %u of %u claimed, %u done, %u parked, %d threads",
k == 0 ? " [" : ";", k, (uint32_t) (h >> 32), (uint32_t) h & 0xffffu, (uint32_t) (h >> 16) & 0xffffu,
ns.done.load(), ns.parked.load(), slice_threads_[(size_t) k]);
s += buf;
}
if (numa_n_ > 0) s += "]";
return s;
}
int ExpertPool::numa_plan_phase(int per, int nb) {
int base = 0;
for (int k = 0; k < numa_n_; ++k) {
NodeSync& ns = nsync_[(size_t) k];
ns.row0 = per * k / numa_n_;
ns.h = per * (k + 1) / numa_n_ - ns.row0;
ns.rows = (int64_t) nb * ns.h;
// about three tasks per thread of the slice's node; an explicit --pool-tasks is shared between the slices
const int64_t want = tasks_ == 0 ? 3 * (int64_t) slice_threads_[(size_t) k] : (int64_t) (std::max)(1, tasks_ / 2);
ns.tasks = (int) (std::min)(want, ns.rows);
ns.base = base;
base += ns.tasks;
}
return base;
}
uint32_t ExpertPool::begin_batch(int n) {
if (n < 0 || n > 0xffff) {
std::fprintf(stderr, "strata: expert pool batch of %d jobs is out of range\n", n);
std::abort();
}
// Every job of the previous batch has completed (`wait_done`), and a claim of it can no longer succeed, so
// nothing adds to `done` until this batch's first claim - which the release below orders after the reset.
const uint32_t e = epoch_.load(std::memory_order_relaxed) + 1; // only the host bumps the epoch
if (numa_phase()) {
// every slice's head, even a slice with no task, and never `head_`: for this epoch only the slice heads
// carry `e`, so a claim on `head_` fails, and every head left from an earlier batch is exhausted (pool.hpp)
for (int k = 0; k < numa_n_; ++k) {
NodeSync& ns = nsync_[(size_t) k];
ns.done.store(0, std::memory_order_relaxed);
ns.drained_ns.store(0, std::memory_order_relaxed);
ns.head.store(pack_head(e, (uint32_t) ns.tasks, 0), std::memory_order_release);
}
pub_ns_ = now_ns();
} else {
done_.store(0, std::memory_order_relaxed);
head_.store(pack_head(e, (uint32_t) n, 0), std::memory_order_release);
}
publish();
return e;
}
void ExpertPool::wait_parked(const char* what) {
hstate_.store(kWaitParked, std::memory_order_relaxed);
hstate_ms_.store(now_ms(), std::memory_order_relaxed);
uint32_t spins = 0;
std::chrono::steady_clock::time_point t0{};
while (parked_count() != (uint32_t) n_) {
strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
const auto now = std::chrono::steady_clock::now();
if (spins == 1024u) t0 = now;
else if (now - t0 > kStall) {
std::fprintf(stderr, "strata: the CPU expert pool stalled %s (%u of %d workers parked)%s - stopping the engine "
"so the server can start it again (issue #29)\n",
what, parked_count(), n_, numa_counters().c_str());
strata::core::release_gpu_waits(stderr); // #267: the GPU may be spinning on this layer's flag
std::fflush(stderr);
std::abort();
}
}
}
void ExpertPool::wait_done(int n) {
hstate_.store(kWaitDone, std::memory_order_relaxed);
hstate_ms_.store(now_ms(), std::memory_order_relaxed);
uint32_t spins = 0, seen = 0;
std::chrono::steady_clock::time_point t0{};
for (;;) {
uint32_t d = 0;
if (numa_phase()) // the slices' counts, each bounded by its tasks: the sum is n only when every slice is done
for (int k = 0; k < numa_n_; ++k) d += nsync_[(size_t) k].done.load(std::memory_order_acquire);
else
d = done_.load(std::memory_order_acquire);
if (d >= (uint32_t) n) return; // `>=`: never a wait that an overshoot outlives
strata_cpu_pause();
if ((++spins & 1023u) != 0) continue;
const auto now = std::chrono::steady_clock::now();
if (spins == 1024u || d != seen) { t0 = now; seen = d; } // progress restarts the clock
else if (now - t0 > kStall) {
std::fprintf(stderr, "strata: the CPU expert pool stalled: %u of %d jobs done, %u of %d workers parked%s - "
"stopping the engine so the server can start it again (issue #29)\n",
d, n, parked_count(), n_, numa_counters().c_str());
strata::core::release_gpu_waits(stderr); // #267
std::fflush(stderr);
std::abort();
}
}
}
void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {
const int s = numa_n_ == 0 ? 0 : id >= 0 ? wslice_[(size_t) id] : hslice_;
for (;;) {
// STRATA_NUMA_SPLIT: a mode-5/6 batch lives on the slice heads, any other on `head_`; the epoch tag makes the
// other family's claim fail, so `mode_` is never read before a claim has succeeded (pool.hpp)
int ci = numa_n_ > 0 ? claim_slice(epoch, s) : -1;
const bool sliced = ci >= 0;
if (!sliced) ci = claim(epoch);
if (ci < 0) break;
const uint32_t i = (uint32_t) ci;
if (id >= 0) wstate_[(size_t) id].store(ci, std::memory_order_relaxed);
else { hstate_.store(ci, std::memory_order_relaxed); hstate_ms_.store(now_ms(), std::memory_order_relaxed); }
if (mode_ == 0) {
const ExpertJob& j = jobs_[i];
s2_expert_vnni_q(j.blob, *j.act, j.out, scratch);
} else if (mode_ == 1) {
const int e = (int) i / parts_a_, part = (int) i % parts_a_;
const int r0 = FF * part / parts_a_, r1 = FF * (part + 1) / parts_a_;
s2_expert_gu_rows(jobs_[e].blob, *jobs_[e].act, split_[(size_t) e].ff, r0, r1);
} else if (mode_ == 2) {
const int e = (int) i / parts_b_, part = (int) i % parts_b_;
const int r0 = H * part / parts_b_, r1 = H * (part + 1) / parts_b_;
s2_expert_down_rows(jobs_[e].blob, split_[(size_t) e].a2, jobs_[e].out, r0, r1);
} else if (mode_ >= 5) {
// plan v0.3 P6: native layers, 5 = gate/up rows, 6 = down rows (the layer's own n_ff / n_embd: Flash-Next's
// FF / H, or another family's smaller expert)
// Task j of T over R rows of `h` per expert from matrix row `row0`: the whole phase (R = mrows_, h = per,
// row0 = 0) or, under the NUMA split, this thread's slice (slice-local row g -> expert g / h, matrix row
// row0 + g % h)
const int per = mode_ == 5 ? (int) nfmt_->n_ff : (int) nfmt_->n_embd;
int64_t R = mrows_, T = mtasks_, j = i;
int h = per, row0 = 0;
if (sliced) {
const NodeSync& ns = nsync_[(size_t) s];
R = ns.rows; T = ns.tasks; h = ns.h; row0 = ns.row0; j = ci - ns.base;
}
const int64_t g0 = R * j / T, g1 = R * (j + 1) / T;
for (int64_t g = g0; g < g1;) {
const int e = (int) (g / h), l0 = (int) (g % h);
const int l1 = (int) std::min<int64_t>(h, l0 + (g1 - g));
const int r0 = row0 + l0, r1 = row0 + l1;
SplitBufMulti& sb = split_multi_[(size_t) e];
const NativeFmt* const nfmt = mjobs_[e].fmt != nullptr ? mjobs_[e].fmt : nfmt_; // 149-cold: same types and rows
if (mode_ == 5 && q2_native_kernels(nfmt->gu_type)) {
// a native Q2_0 pack: gate and up rows on the Q2_0 kernels, then SwiGLU
thread_local float gbuf[MAXT][FF], ubuf[MAXT][FF];
float* gp[MAXT];
float* up[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) { gp[t] = gbuf[t]; up[t] = ubuf[t]; }
const int nbk = (int) (nfmt->n_embd / 64);
q2_rows_any(mjobs_[e].blob, nfmt->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, gp, r0, r1);
q2_rows_any(mjobs_[e].blob + nfmt->up_off, nfmt->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, up, r0, r1);
for (int t = 0; t < mjobs_[e].nt; ++t)
for (int r = r0; r < r1; ++r)
sb.ff[t][r] = (gbuf[t][r] / (1.f + std::exp(-gbuf[t][r]))) * ubuf[t][r];
} else if (mode_ == 5) {
float* ff[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) ff[t] = sb.ff[t];
if (narithmetic_ == NativeArithmetic::WindowInvariant)
native_gu_rows_invariant(*nfmt, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);
else
native_gu_rows(*nfmt, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);
} else if (q2_native_kernels(nfmt->d_type)) {
// Q2_0 down (most IQ layers): the AVX-512 kernel, ggml-cpu has only a scalar one on x86
const ActQ* a2[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) a2[t] = &sb.a2[t];
q2_rows_any(mjobs_[e].blob + nfmt->down_off, nfmt->d_row, (int) (nfmt->n_ff / 64), a2,
mjobs_[e].nt, mjobs_[e].out, r0, r1);
} else {
const void* hq[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) hq[t] = sb.hq[t];
if (narithmetic_ == NativeArithmetic::WindowInvariant)
native_down_rows_invariant(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);
else
native_down_rows(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);
}
g += l1 - l0;
}
} else {
// plan v0.3 P6: an equal range of the phase's rows across ALL its experts (a range may span two)
const int per = mode_ == 3 ? FF : H;
const int64_t g0 = mrows_ * (int64_t) i / mtasks_, g1 = mrows_ * (int64_t) (i + 1) / mtasks_;
for (int64_t r = g0; r < g1;) {
const int e = (int) (r / per), r0 = (int) (r % per);
const int r1 = (int) std::min<int64_t>(per, r0 + (g1 - r));
SplitBufMulti& sb = split_multi_[(size_t) e];
if (mode_ == 3) {
float* ff[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) ff[t] = sb.ff[t];
s2_expert_gu_rows_multi(mjobs_[e].blob, mjobs_[e].act, mjobs_[e].nt, ff, r0, r1);
} else {
const ActQ* a2[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) a2[t] = &sb.a2[t];
s2_expert_down_rows_multi(mjobs_[e].blob, a2, mjobs_[e].nt, mjobs_[e].out, r0, r1);
}
r += r1 - r0;
}
}
if (sliced) {
NodeSync& ns = nsync_[(size_t) s];
const uint32_t t = (uint32_t) ns.tasks;
// the slice's last task stamps its drain time; read by the host once this thread has parked again
if (ns.done.fetch_add(1, std::memory_order_release) + 1 == t)
ns.drained_ns.store(now_ns(), std::memory_order_relaxed);
} else {
done_.fetch_add(1, std::memory_order_release);
}
}
}
void ExpertPool::run_phase(int mode, int n_tasks) {
wait_parked("before a phase");
mode_ = mode;
njobs_ = n_tasks;
const uint32_t e = begin_batch(n_tasks);
if (host_works_) drain(-1, host_scratch_, e);
wait_done(n_tasks);
wait_parked("after a phase");
// Each slice's stamp is stored after its last `done` increment, so `wait_done` can return before it lands; the
// thread that stored it has parked again since (it left the park to claim), and that re-park is what the wait
// above acquired. So the stamps are read here, not in `wait_done`.
if (numa_phase())
for (int k = 0; k < numa_n_; ++k) {
const int64_t t = nsync_[(size_t) k].drained_ns.load(std::memory_order_acquire);
if (t != 0) slice_ms_[k] += (double) (t - pub_ns_) / 1e6;
}
hstate_.store(kIdle, std::memory_order_relaxed);
hstate_ms_.store(now_ms(), std::memory_order_relaxed);
}
void ExpertPool::run_split(ExpertJob* jobs, int n) {
if (n <= 0) return;
if (n > kMaxSplit || n_ == 1 || expert_oracle_q8_0_enabled()) { run(jobs, n); return; }
const auto t0 = std::chrono::steady_clock::now();
jobs_ = jobs;
const int threads = n_ + (host_works_ ? 1 : 0);
// about three tasks per thread in each phase, so the tail is short
parts_a_ = (std::max)(1, (3 * threads + n - 1) / n);
parts_b_ = parts_a_;
run_phase(1, n * parts_a_);
for (int e = 0; e < n; ++e) act_quant_q8_1(split_[(size_t) e].ff, FF, split_[(size_t) e].a2);
run_phase(2, n * parts_b_);
mode_ = 0;
ms_drain_ += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
}
int ExpertPool::phase_tasks(int64_t rows) const {
return tasks_ == 0 ? 3 * (n_ + (host_works_ ? 1 : 0)) : (int) (std::min)(rows, (int64_t) tasks_);
}
void ExpertPool::run_split_multi(ExpertJobMulti* jobs, int n) {
if (n <= 0) return;
if (n > kMaxSplitMulti || expert_oracle_q8_0_enabled()) {
// one token at a time through the single-token path (the oracle contract has no multi kernel)
std::vector<ExpertJob> single;
for (int e = 0; e < n; ++e)
for (int t = 0; t < jobs[e].nt; ++t) {
ExpertJob j;
j.blob = jobs[e].blob;
j.act = jobs[e].act[t];
j.out = jobs[e].out[t];
single.push_back(j);
}
for (size_t i = 0; i < single.size(); i += kMaxSplit)
run_split(single.data() + i, (int) (std::min)((size_t) kMaxSplit, single.size() - i));
return;
}
const auto t0 = std::chrono::steady_clock::now();
mjobs_ = jobs;
mrows_ = (int64_t) n * FF;
mtasks_ = phase_tasks(mrows_);
run_phase(3, mtasks_);
const auto t1 = std::chrono::steady_clock::now();
for (int e = 0; e < n; ++e)
for (int t = 0; t < jobs[e].nt; ++t)
act_quant_q8_1(split_multi_[(size_t) e].ff[t], FF, split_multi_[(size_t) e].a2[t]);
const auto t2 = std::chrono::steady_clock::now();
mrows_ = (int64_t) n * H;
mtasks_ = phase_tasks(mrows_);
run_phase(4, mtasks_);
const auto t3 = std::chrono::steady_clock::now();
ms_multi_gu += std::chrono::duration<double, std::milli>(t1 - t0).count();
ms_multi_q += std::chrono::duration<double, std::milli>(t2 - t1).count();
ms_multi_down += std::chrono::duration<double, std::milli>(t3 - t2).count();
multi_bytes += (int64_t) n * (int64_t) BLOB;
mode_ = 0;
ms_drain_ += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
}
void ExpertPool::run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobs, int n, NativeArithmetic arithmetic) {
if (n <= 0) return;
const auto t0 = std::chrono::steady_clock::now();
// more distinct experts than buffers: run them in batches
for (int b0 = 0; b0 < n; b0 += kMaxSplitMulti) {
const int nb = (std::min)(kMaxSplitMulti, n - b0);
mjobs_ = jobs + b0;
nfmt_ = &f;
narithmetic_ = arithmetic;
mrows_ = (int64_t) nb * f.n_ff;
mtasks_ = numa_n_ > 0 ? numa_plan_phase((int) f.n_ff, nb) : phase_tasks(mrows_);
const auto a = std::chrono::steady_clock::now();
run_phase(5, mtasks_);
const auto b = std::chrono::steady_clock::now();
for (int e = 0; e < nb; ++e)
for (int t = 0; t < mjobs_[e].nt; ++t)
if (q2_native_kernels(f.d_type)) act_quant_any(split_multi_[(size_t) e].ff[t], FF, split_multi_[(size_t) e].a2[t]);
else native_quant_h(f, split_multi_[(size_t) e].ff[t], split_multi_[(size_t) e].hq[t]);
const auto c = std::chrono::steady_clock::now();
mrows_ = (int64_t) nb * f.n_embd;
mtasks_ = numa_n_ > 0 ? numa_plan_phase((int) f.n_embd, nb) : phase_tasks(mrows_);
run_phase(6, mtasks_);
const auto d = std::chrono::steady_clock::now();
ms_multi_gu += std::chrono::duration<double, std::milli>(b - a).count();
ms_multi_q += std::chrono::duration<double, std::milli>(c - b).count();
ms_multi_down += std::chrono::duration<double, std::milli>(d - c).count();
}
multi_bytes += (int64_t) n * (int64_t) f.bytes;
mode_ = 0;
ms_drain_ += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
}
void ExpertPool::run(ExpertJob* jobs, int n) {
if (n <= 0) return;
if (n_ == 1) { // no workers: run inline, so a single-core machine still produces a token
for (int i = 0; i < n; ++i) s2_expert_vnni_q(jobs[i].blob, *jobs[i].act, jobs[i].out, scratch_[0]);
return;
}
// Wait for every worker to be parked BEFORE touching the batch, so the publish below is the only thing
// that can move a worker into the drain loop.
//
// THE THREE PHASES ARE TIMED SEPARATELY. They were one number, which cannot distinguish a pool that is
// slow at the WORK from one that is slow at the SYNCHRONISATION - and those need opposite fixes.
const auto t_a = std::chrono::steady_clock::now();
wait_parked("before a batch");
const auto t_b = std::chrono::steady_clock::now();
jobs_ = jobs;
njobs_ = n;
mode_ = 0;
const uint32_t e = begin_batch(n); // done, then head (release), then the epoch: the batch is described first
// ---- **THE HOST DRAINS TOO (R2.2), INSTEAD OF SPINNING ON `done_`.**
//
// The loop below used to be `while (done_ != n) strata_cpu_pause();`. The host is pinned to core 0 - the core
// `physical_cores(true)` deliberately keeps the five workers off - so for the whole drain that core was
// idle while five cores did six cores' worth of work. Measured before the change: 33.7 GB/s against
// 5/6 x 44.14 = 36.8 for five workers and 44.14 for six.
//
// The host claims through the SAME `head_` counter, so this is not a second scheduler and nothing about
// the ordering changes: `head_` is a single `fetch_add`, every job is the same size, and a thread that
// arrives late simply claims nothing. `done_` is still the completion signal and the host still waits for
// it - what changed is only that the host arrives at that wait having done a share of the work.
//
// The host's `done_.fetch_add` is a release for the same reason a worker's is: `j.out` is read by the
// device after `run()` returns, so the write must be published, not merely performed.
if (host_works_) {
for (;;) {
const int ci = claim(e);
if (ci < 0) break;
hstate_.store(ci, std::memory_order_relaxed);
hstate_ms_.store(now_ms(), std::memory_order_relaxed);
const ExpertJob& j = jobs_[ci];
s2_expert_vnni_q(j.blob, *j.act, j.out, host_scratch_);
done_.fetch_add(1, std::memory_order_release);
}
}
wait_done(n);
// And park again, so the next `run` starts from a known state. See the header for why `done` alone is
// not enough.
const auto t_c = std::chrono::steady_clock::now();
wait_parked("after a batch");
hstate_.store(kIdle, std::memory_order_relaxed);
hstate_ms_.store(now_ms(), std::memory_order_relaxed);
const auto t_d = std::chrono::steady_clock::now();
ms_wait_park_ += std::chrono::duration<double, std::milli>(t_b - t_a).count();
ms_drain_ += std::chrono::duration<double, std::milli>(t_c - t_b).count();
ms_repark_ += std::chrono::duration<double, std::milli>(t_d - t_c).count();
}
} // namespace strata::kernels::cpu
#if defined(_WIN32)
// strata/platform/thread_name.hpp: the Windows half lives here, where <windows.h> is already included. SetThreadDescription
// (Windows 10 1607 and later) is looked up, not linked: older SDKs and toolchains lack the import.
namespace strata::platform {
void name_this_thread(const char* base, int index) {
char name[64];
if (index >= 0) std::snprintf(name, sizeof name, "%s-%d", base, index);
else std::snprintf(name, sizeof name, "%s", base);
using SetDesc = HRESULT(WINAPI*)(HANDLE, PCWSTR);
static const SetDesc fn = (SetDesc) GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "SetThreadDescription");
if (fn == nullptr) return;
wchar_t wide[64];
int i = 0;
for (; name[i] != '\0' && i < 63; ++i) wide[i] = (wchar_t) (unsigned char) name[i];
wide[i] = L'\0';
(void) fn(GetCurrentThread(), wide);
}
} // namespace strata::platform
#endif