Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

fix: interleave an explicit pool worker count across sockets when the NUMA split is on

Baptiste Rajaut5b8cdaf2d97b
diff --git a/include/strata/kernels/cpu/pool_plan.hpp b/include/strata/kernels/cpu/pool_plan.hpp
index c6a53cf34..42e538bf1 100644
--- a/include/strata/kernels/cpu/pool_plan.hpp
+++ b/include/strata/kernels/cpu/pool_plan.hpp
@@ -145,8 +145,13 @@ inline std::string pool_workers_warning(int requested, const PoolPlanInput& in,
std::to_string(plan.workers) + " workers and the end of each layer waits for every one). " + rec;
}
-/// STRATA_NUMA_SPLIT: the order to take worker cores in so that a default count below the core count still spans the
-/// packages evenly - round-robin over the packages in their first-seen order (the host's first), stable inside each.
+/// STRATA_NUMA_SPLIT: whether two slices of `a` and `b` threads may split the rows 50/50 - a slice with under half
+/// the other's threads would be the straggler of every batch, so the pool refuses that split.
+inline bool numa_slices_balanced(int a, int b) { return std::min(a, b) * 2 >= std::max(a, b); }
+
+/// STRATA_NUMA_SPLIT: the order to take worker cores in so that a count below the core count (the default or a
+/// configured one) still spans the packages evenly - round-robin over the packages in their first-seen order (the
+/// host's first), stable inside each.
/// `package[i]` is worker core i's package (-1: unknown, a package of its own); returns indices into it.
inline std::vector<size_t> interleave_packages(const std::vector<int>& package) {
std::vector<int> ids; // packages in first-seen order
diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cpp
index 12abbcc00..d2a91fb9d 100644
--- a/src/kernels/cpu/pool.cpp
+++ b/src/kernels/cpu/pool.cpp
@@ -656,25 +656,26 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
const bool env_known = !env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0);
const PoolPlanInput plan_in = pool_plan_input(topo_);
const PoolPlan wplan = plan_pool_workers(plan_in);
+ // the split needs workers on both nodes: its cores are taken round-robin over the packages, so any count under
+ // the core count (the default or a configured one) spans them evenly (one package: the order is kept)
+ if (env_known && topo_.worker_package.size() == topo_.worker_cores.size()) {
+ std::vector<int> cores, pkgs;
+ for (size_t i : interleave_packages(topo_.worker_package)) {
+ cores.push_back(topo_.worker_cores[i]);
+ pkgs.push_back(topo_.worker_package[i]);
+ }
+ topo_.worker_cores = std::move(cores);
+ topo_.worker_package = std::move(pkgs);
+ }
if (n_workers > 0) {
n_ = n_workers;
} else if (topo_.is_hybrid && affinity_ != PoolAffinity::All) {
n_ = (std::max)(1, topo_.p_cores - 1);
} else if (env_known) {
- // the split needs workers on both nodes: the default spans every socket (the plan's other bounds stay), its
- // cores taken round-robin over the packages so a count under the core count still spans them evenly
+ // ... and the default spans every socket (the plan's other bounds stay)
PoolPlanInput span_in = plan_in;
span_in.span_packages = true;
const PoolPlan sp = plan_pool_workers(span_in);
- if (topo_.worker_package.size() == topo_.worker_cores.size()) {
- std::vector<int> cores, pkgs;
- for (size_t i : interleave_packages(topo_.worker_package)) {
- cores.push_back(topo_.worker_cores[i]);
- pkgs.push_back(topo_.worker_package[i]);
- }
- topo_.worker_cores = std::move(cores);
- topo_.worker_package = std::move(pkgs);
- }
n_ = (std::min)(sp.workers, (std::max)(1, (int) topo_.worker_cores.size()));
plan_limit_ = sp.limit;
} else {
@@ -743,8 +744,7 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
plan.cores[slice[(size_t) j]].push_back(core_of(j));
++per[slice[(size_t) j]];
}
- // rows stay 50/50, so a slice with under half the other's threads would be the straggler of every batch
- if (std::min(per[0], per[1]) * 2 < std::max(per[0], per[1]))
+ if (!numa_slices_balanced(per[0], per[1]))
std::snprintf(why, sizeof why, "slices too unbalanced (node %d: %d thread%s, node %d: %d thread%s)",
nodes[0], per[0], per[0] == 1 ? "" : "s", nodes[1], per[1], per[1] == 1 ? "" : "s");
}
diff --git a/src/kernels/cpu/pool_plan_test.cpp b/src/kernels/cpu/pool_plan_test.cpp
index 7ceff0387..3d7add149 100644
--- a/src/kernels/cpu/pool_plan_test.cpp
+++ b/src/kernels/cpu/pool_plan_test.cpp
@@ -183,6 +183,26 @@ int main() {
const std::vector<size_t> same = cpu::interleave_packages({1, 1, 1});
expect(same == std::vector<size_t>({0, 1, 2}), "one package: the order is kept");
}
+ // a configured count under the split, on the 2 x 12-core box: the worker cores as fill_plan_fields orders them
+ // (the host's socket first: its 11 cores but the host's, then the other socket's 12), taken in interleave order;
+ // the host thread drains on socket 0
+ {
+ std::vector<int> pkg(11, 0);
+ pkg.insert(pkg.end(), 12, 1);
+ const std::vector<size_t> o = cpu::interleave_packages(pkg);
+ auto per = [&](int n, bool interleaved, int s) {
+ int c = 0;
+ for (int i = 0; i < n; ++i) c += pkg[interleaved ? o[(size_t) i] : (size_t) i] == s;
+ return c;
+ };
+ expect(per(12, true, 0) == 6 && per(12, true, 1) == 6, "--pool-workers 12 under the split: 6 + 6 workers");
+ expect(cpu::numa_slices_balanced(per(12, true, 0) + 1, per(12, true, 1)), "12: 7 / 6 threads, balanced");
+ expect(per(13, true, 0) == 7 && per(13, true, 1) == 6, "--pool-workers 13 under the split: 7 + 6 workers");
+ expect(cpu::numa_slices_balanced(per(13, true, 0) + 1, per(13, true, 1)), "13: 8 / 6 threads, balanced");
+ expect(!cpu::numa_slices_balanced(per(12, false, 0) + 1, per(12, false, 1)),
+ "12 in the host-first order would be 12 / 1 threads: refused (what the interleave avoids)");
+ expect(per(22, true, 0) == 11 && per(22, true, 1) == 11, "22 (the split default here): 11 + 11 workers");
+ }
if (g_fail) {
std::fprintf(stderr, "pool_plan_test: %d failure(s)\n", g_fail);