Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

feat: span both sockets with the default pool workers when the NUMA split is on

Baptiste Rajautea6b385df86d
diff --git a/include/strata/kernels/cpu/pool_plan.hpp b/include/strata/kernels/cpu/pool_plan.hpp
index d53f45798..c6a53cf34 100644
--- a/include/strata/kernels/cpu/pool_plan.hpp
+++ b/include/strata/kernels/cpu/pool_plan.hpp
@@ -43,6 +43,10 @@ struct PoolPlanInput {
/// +12.6% over the 19 of one-per-core, 24 workers -44%: notes-152-users.md).
int p_cores = 0;
int e_cores = 0;
+ /// STRATA_NUMA_SPLIT (opt-in): the split needs workers on both NUMA nodes, so the default may use the cores of
+ /// every package instead of the host's only, and a configured count above the host's socket is not warned about.
+ /// The other bounds (reserve, hybrid, quota, cap) stay. False: the plan above, unchanged.
+ bool span_packages = false;
};
/// #3262: is a CPU an efficiency-class core? `pmu_core` / `pmu_atom` / `pmu_lowpower` are Intel's hybrid PMU lists
@@ -83,11 +87,14 @@ inline PoolPlan plan_pool_workers(const PoolPlanInput& in) {
local = cores;
p.packages = 1;
}
+ const bool span = in.span_packages && p.topology_known && p.packages > 1;
+ if (span) local = cores; // STRATA_NUMA_SPLIT: every socket counts as local
p.cores = cores;
p.cores_local = local;
p.limit = "every physical core but the host's";
int n = std::max(1, local - 1); // the host thread has the first core
if (!p.topology_known) p.limit = "half the logical processors (the OS gave no core layout)";
+ else if (span) p.limit = "every CPU socket (STRATA_NUMA_SPLIT)";
else if (p.packages > 1) p.limit = "the host's CPU socket";
if (n >= kPoolReserveFrom) { // a big CPU keeps a core for the engine's other threads
n -= kPoolReserve;
@@ -127,7 +134,7 @@ inline std::string pool_workers_warning(int requested, const PoolPlanInput& in,
if (plan.topology_known && requested + 1 > plan.cores)
return asked + ", but this CPU has " + std::to_string(plan.cores) +
" physical cores. Two workers on one core share its memory bandwidth and slow each other down. " + rec;
- if (plan.packages > 1 && requested + 1 > plan.cores_local)
+ if (plan.packages > 1 && !in.span_packages && requested + 1 > plan.cores_local)
return asked + ", more than the " + std::to_string(plan.cores_local) +
" cores of the CPU socket the engine's main thread runs on. Workers on the other socket read the experts " +
"across the link and were measured slower than fewer local ones. " + rec;
@@ -138,4 +145,22 @@ inline std::string pool_workers_warning(int requested, const PoolPlanInput& in,
std::to_string(plan.workers) + " workers and the end of each layer waits for every one). " + rec;
}
+/// STRATA_NUMA_SPLIT: the order to take worker cores in so that a default count below the core count still spans the
+/// packages evenly - round-robin over the packages in their first-seen order (the host's first), stable inside each.
+/// `package[i]` is worker core i's package (-1: unknown, a package of its own); returns indices into it.
+inline std::vector<size_t> interleave_packages(const std::vector<int>& package) {
+ std::vector<int> ids; // packages in first-seen order
+ std::vector<std::vector<size_t>> of; // each package's worker indices, in order
+ for (size_t i = 0; i < package.size(); ++i) {
+ const size_t k = (size_t) (std::find(ids.begin(), ids.end(), package[i]) - ids.begin());
+ if (k == ids.size()) { ids.push_back(package[i]); of.emplace_back(); }
+ of[k].push_back(i);
+ }
+ std::vector<size_t> order;
+ for (size_t r = 0; order.size() < package.size(); ++r)
+ for (const auto& v : of)
+ if (r < v.size()) order.push_back(v[r]);
+ return order;
+}
+
} // namespace strata::kernels::cpu
diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cpp
index f39f5efae..12abbcc00 100644
--- a/src/kernels/cpu/pool.cpp
+++ b/src/kernels/cpu/pool.cpp
@@ -650,16 +650,33 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
if (tasks < 0 || tasks > kMaxTasks) throw std::invalid_argument("expert pool tasks must be in 0..4096");
if (const char* e = std::getenv("STRATA_POOL_SPIN_US")) // a test knob; see kSpinBeforeSleep
spin_before_sleep_ = std::chrono::microseconds((std::max)(0, std::atoi(e)));
+ // STRATA_NUMA_SPLIT, read here for the worker count (the split itself is decided below, once the count is known)
+ const char* env = std::getenv("STRATA_NUMA_SPLIT");
+ const bool env_off = env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0;
+ const bool env_known = !env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0);
const PoolPlanInput plan_in = pool_plan_input(topo_);
const PoolPlan wplan = plan_pool_workers(plan_in);
if (n_workers > 0) {
n_ = n_workers;
- // recommend, never force: a count above what this CPU uses well runs as configured, with a plain warning
- static std::atomic<bool> warned{false};
- const std::string why = pool_workers_warning(n_workers, plan_in, wplan);
- if (!why.empty() && !warned.exchange(true)) std::fprintf(stderr, "strata cpu pool: warning: %s\n", why.c_str());
} else if (topo_.is_hybrid && affinity_ != PoolAffinity::All) {
n_ = (std::max)(1, topo_.p_cores - 1);
+ } else if (env_known) {
+ // the split needs workers on both nodes: the default spans every socket (the plan's other bounds stay), its
+ // cores taken round-robin over the packages so a count under the core count still spans them evenly
+ PoolPlanInput span_in = plan_in;
+ span_in.span_packages = true;
+ const PoolPlan sp = plan_pool_workers(span_in);
+ if (topo_.worker_package.size() == topo_.worker_cores.size()) {
+ std::vector<int> cores, pkgs;
+ for (size_t i : interleave_packages(topo_.worker_package)) {
+ cores.push_back(topo_.worker_cores[i]);
+ pkgs.push_back(topo_.worker_package[i]);
+ }
+ topo_.worker_cores = std::move(cores);
+ topo_.worker_package = std::move(pkgs);
+ }
+ n_ = (std::min)(sp.workers, (std::max)(1, (int) topo_.worker_cores.size()));
+ plan_limit_ = sp.limit;
} else {
n_ = (std::min)(wplan.workers, (std::max)(1, (int) topo_.worker_cores.size()));
plan_limit_ = wplan.limit;
@@ -678,9 +695,6 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
// STRATA_NUMA_SPLIT: decided here, before any worker starts (a worker counts into its slice's `parked` from its
// first instruction), and never revised. Unset: the plan stays empty and nothing below runs.
NumaSplitPlan plan;
- const char* env = std::getenv("STRATA_NUMA_SPLIT");
- const bool env_off = env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0;
- const bool env_known = !env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0);
if (!env_off && !env_known) {
plan.why = std::string("unknown STRATA_NUMA_SPLIT value '") + env + "' (use 1 or test)";
std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());
@@ -763,6 +777,15 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());
}
}
+ if (n_workers > 0) {
+ // recommend, never force: a count above what this CPU uses well runs as configured, with a plain warning
+ // (STRATA_NUMA_SPLIT on: workers on the other socket are the point, that case is not warned about)
+ PoolPlanInput warn_in = plan_in;
+ warn_in.span_packages = plan.on;
+ static std::atomic<bool> warned{false};
+ const std::string why = pool_workers_warning(n_workers, warn_in, plan_pool_workers(warn_in));
+ if (!why.empty() && !warned.exchange(true)) std::fprintf(stderr, "strata cpu pool: warning: %s\n", why.c_str());
+ }
g_numa_plan = std::move(plan);
g_numa_owner = this;
threads_.reserve((size_t) n_);
diff --git a/src/kernels/cpu/pool_plan_test.cpp b/src/kernels/cpu/pool_plan_test.cpp
index a30fb98f3..7ceff0387 100644
--- a/src/kernels/cpu/pool_plan_test.cpp
+++ b/src/kernels/cpu/pool_plan_test.cpp
@@ -150,6 +150,40 @@ int main() {
expect(cpu::pool_workers_warning(8, q, cpu::plan_pool_workers(q)).find("container") != std::string::npos,
"8 workers under a 4-CPU quota: names the container limit");
+ // STRATA_NUMA_SPLIT (span_packages): the default spans every socket, the other bounds stay; off = the plan above
+ {
+ auto x = xeon; // 2 x 20 cores: 39 but the host's, one kept free -> 38, then the cap
+ x.span_packages = true;
+ line("2-socket 40c/80t, NUMA split", x);
+ const cpu::PoolPlan px = cpu::plan_pool_workers(x);
+ expect(px.workers == cpu::kPoolWorkersCap, "2x20c split: both sockets, then the cap");
+ expect(px.cores_local == 40, "2x20c split: every socket counts as local");
+ auto b = cpu_of(48, {12, 12}); // the 2 x Xeon Gold 6226 box: 23 but the host's, one kept free
+ expect(cpu::plan_pool_workers(b).workers == 11, "2x12c no split: the host socket's 11 (unchanged)");
+ b.span_packages = true;
+ expect(cpu::plan_pool_workers(b).workers == 22, "2x12c split: 22 workers across both sockets");
+ auto one = cpu_of(16, {8});
+ one.span_packages = true;
+ expect(cpu::plan_pool_workers(one).workers == 7, "one socket: the split changes nothing");
+ // the socket warning is silenced while the split is on; the others stay
+ expect(cpu::pool_workers_warning(30, x, px).empty(), "30 workers on 2x20c under the split: no socket warning");
+ auto bx = cpu_of(80, {20, 20});
+ bx.span_packages = true;
+ const std::string w = cpu::pool_workers_warning(36, bx, cpu::plan_pool_workers(bx));
+ expect(!w.empty() && w.find("socket") == std::string::npos, "36 on 2x20c under the split: the cap warning, not the socket one");
+ expect(cpu::pool_workers_warning(60, bx, cpu::plan_pool_workers(bx)).find("physical cores") != std::string::npos,
+ "60 on 2x20c under the split: still more than the physical cores");
+ }
+ // interleave_packages: round-robin over the packages (host's first), stable inside each
+ {
+ const std::vector<size_t> o = cpu::interleave_packages({0, 0, 0, 1, 1, 1, 1});
+ const std::vector<size_t> want = {0, 3, 1, 4, 2, 5, 6};
+ expect(o == want, "interleave 3+4: 0 3 1 4 2 5 6");
+ expect(cpu::interleave_packages({}).empty(), "interleave of nothing");
+ const std::vector<size_t> same = cpu::interleave_packages({1, 1, 1});
+ expect(same == std::vector<size_t>({0, 1, 2}), "one package: the order is kept");
+ }
+
if (g_fail) {
std::fprintf(stderr, "pool_plan_test: %d failure(s)\n", g_fail);
return 1;
diff --git a/src/kernels/cpu/pool_tasks_test.cpp b/src/kernels/cpu/pool_tasks_test.cpp
index 8267e52fe..d1955c382 100644
--- a/src/kernels/cpu/pool_tasks_test.cpp
+++ b/src/kernels/cpu/pool_tasks_test.cpp
@@ -240,6 +240,20 @@ int main(int argc, char** argv) {
}
unset_env("STRATA_NUMA_SPLIT");
if (!ok) { std::fprintf(stderr, "numa-split real-mode plan invariant broken\n"); return 1; }
+ // the DEFAULT worker count under the split spans every socket (pool_plan.hpp span_packages): on a 2-socket
+ // host the split must then be on, where 0.2.0's host-socket default alone would leave every thread on one node
+ put_env("STRATA_NUMA_SPLIT", "1");
+ {
+ const int packages = (int) cpu::pool_plan_input(cpu::detect_cpu_topology(true, cpu::PoolAffinity::All))
+ .cores_per_package.size();
+ cpu::ExpertPool pool(0, true, true);
+ const auto& plan = cpu::active_numa_split();
+ std::printf("pool tasks: default pool under the split: %d workers on %d socket(s), split %s%s (%s)\n",
+ pool.workers(), packages, plan.on ? "on" : "off: ", plan.why.c_str(), pool.plan_limit());
+ ok = packages != 2 || (plan.on && plan.cores[0].size() > 0 && plan.cores[1].size() > 0);
+ }
+ unset_env("STRATA_NUMA_SPLIT");
+ if (!ok) { std::fprintf(stderr, "numa-split: the default pool on a 2-socket host did not span both nodes\n"); return 1; }
// a value other than 1 / test is off, with the value in the reason; "0" and "" are off silently
for (const char* v : {"off", "yes"}) {
put_env("STRATA_NUMA_SPLIT", v);