Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
NUMA pool: tried by itself on a machine with 2 or more NUMA nodes, STRATA_NUMA_POOL=0 turns it off, one node unchanged
Niko1221e4d22e7ef112
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 0a28c077a..1f2a6a59a 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -2112,6 +2112,11 @@ if(STRATA_BUILD_TESTS)add_executable(pool_plan_test src/kernels/cpu/pool_plan_test.cpp)target_include_directories(pool_plan_test PRIVATE include)add_test(NAME pool_plan_test COMMAND pool_plan_test)+ # 0.2.2: when the pool splits per NUMA node by itself (a fake sysfs node folder, mock cores; STRATA_NUMA_POOL=0, one node = unchanged)+ add_executable(numa_pool_test src/kernels/cpu/numa_pool_test.cpp)+ target_include_directories(numa_pool_test PRIVATE include)+ target_compile_features(numa_pool_test PRIVATE cxx_std_17)+ add_test(NAME numa_pool_test COMMAND numa_pool_test)# #3204: when the HIP start-up warm-up of hipBLAS may run (pure policy, no GPU)add_executable(warm_policy_test src/prefill/warm_policy_test.cpp)target_include_directories(warm_policy_test PRIVATE include)diff --git a/include/strata/kernels/cpu/numa_pool.hpp b/include/strata/kernels/cpu/numa_pool.hppnew file mode 100644index 000000000..5db33846a--- /dev/null+++ b/include/strata/kernels/cpu/numa_pool.hpp@@ -0,0 +1,152 @@+// include/strata/kernels/cpu/numa_pool.hpp - when the CPU expert pool splits per NUMA node (0.2.2).+//+// Pure decisions, no threads and no globals, so a test can feed them a made-up machine:+// * numa_cpu_node_count_at(dir) how many NUMA nodes with CPUs a sysfs node folder lists (Linux: /sys/devices/system/node;+// Windows reads GetNumaHighestNodeNumber in pool.cpp; every other system: 1)+// * pick_numa_pool(...) whether the split is tried without STRATA_NUMA_SPLIT, and the line that says so+// * decide_numa_split(...) the split's plan for a pool whose threads sit on given cores+//+// One node is exactly the code path before the split existed: pick_numa_pool answers `Unchanged` with no log line and the pool+// never reads anything else from here. STRATA_NUMA_POOL=0 is the off switch of the default; STRATA_NUMA_SPLIT=1 / test keep+// their meaning from the contributor's branch (#3201).+#pragma once++#include "strata/kernels/cpu/pool_plan.hpp"++#include <algorithm>+#include <cctype>+#include <cstdlib>+#include <cstring>+#include <filesystem>+#include <fstream>+#include <functional>+#include <string>+#include <system_error>+#include <vector>++namespace strata::kernels::cpu {++/// Whether text is "0", "off", "false" or "no" (any case): the spelling of every off switch of the pool.+inline bool numa_switch_off(const char* v) {+ if (v == nullptr) return false;+ std::string s(v);+ std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return (char) std::tolower(c); });+ return s == "0" || s == "off" || s == "false" || s == "no";+}++/// The NUMA nodes with CPUs under a sysfs node folder (`node0`, `node1`, ... each holding a `cpulist`). A node without CPUs+/// (CXL or other memory-only nodes) is not counted: the pool splits threads, and a node with none has none to split.+/// A folder that is missing or has no node entries counts 0 (the caller treats 0 and 1 alike: no split).+inline int numa_cpu_node_count_at(const std::string& node_dir) {+ std::error_code ec;+ int nodes = 0;+ for (std::filesystem::directory_iterator it(node_dir, ec), end; !ec && it != end; it.increment(ec)) {+ const std::string name = it->path().filename().string();+ if (name.size() < 5 || name.compare(0, 4, "node") != 0) continue;+ if (!std::all_of(name.begin() + 4, name.end(), [](unsigned char c) { return std::isdigit(c) != 0; })) continue;+ std::ifstream f(it->path() / "cpulist");+ std::string list;+ std::getline(f, list);+ if (list.find_first_of("0123456789") != std::string::npos) ++nodes;+ }+ return nodes;+}++enum class NumaPoolChoice {+ Unchanged, ///< nothing new: STRATA_NUMA_SPLIT (set or not) decides as it did before 0.2.2+ Off, ///< STRATA_NUMA_POOL=0 on a machine that matters: the split is off, whatever STRATA_NUMA_SPLIT says+ Auto, ///< two or more NUMA nodes and nothing set: try the split by itself; if it cannot run the pool stays as it was+};+struct NumaPoolPick {+ NumaPoolChoice choice = NumaPoolChoice::Unchanged;+ std::string log; ///< the line to print ("" = print nothing)+};++/// `pool_env` is STRATA_NUMA_POOL, `split_env` STRATA_NUMA_SPLIT (nullptr or "" = unset), `nodes` the NUMA nodes with CPUs.+/// one node, nothing set -> Unchanged, silent (today's code path to the instruction)+/// STRATA_NUMA_POOL=0, two or more nodes -> Off, with a line that says so+/// STRATA_NUMA_POOL=0, STRATA_NUMA_SPLIT set -> Off (the off switch wins), with a line+/// STRATA_NUMA_SPLIT set (1, test, 0, ...) -> Unchanged: the contributor's variable keeps its old meaning+/// two or more nodes, nothing set -> Auto+inline NumaPoolPick pick_numa_pool(const char* pool_env, const char* split_env, int nodes) {+ const bool split_set = split_env != nullptr && *split_env != '\0';+ NumaPoolPick pick;+ if (numa_switch_off(pool_env) && (nodes >= 2 || split_set)) {+ pick.choice = NumaPoolChoice::Off;+ pick.log = "strata cpu pool: NUMA pool off (STRATA_NUMA_POOL=0): " ++ (nodes >= 2 ? std::to_string(nodes) + " NUMA nodes found" : std::string("one NUMA node")) ++ ", the pool runs as before";+ return pick;+ }+ if (split_set || nodes < 2) return pick;+ pick.choice = NumaPoolChoice::Auto;+ return pick;+}++struct NumaSplitDecision {+ bool on = false;+ std::string why; ///< the reason when off+ int nodes[2] = {-1, -1}; ///< the two node ids, ascending+ std::vector<int> slice; ///< each thread's slice (0 or 1)+ int per[2] = {0, 0}; ///< threads per slice+};++/// The real-node split of a pool whose threads sit on `thread_core` (workers first, then the host when it drains; -1 = no core).+/// `n_workers` tells a worker from the host in the messages; `pinned` is whether the pool pins its threads; `rotate` is+/// STRATA_EXCHANGE_ROTATE=1 (experts move between slots, the split needs fixed ones); `node_of(core)` is the core's NUMA node or -1.+inline NumaSplitDecision decide_numa_split(const std::vector<int>& thread_core, int n_workers, bool pinned, bool rotate,+ const std::function<int(int)>& node_of) {+ NumaSplitDecision d;+ const int threads = (int) thread_core.size();+ std::vector<int> node((size_t) threads, -1);+ if (threads < 2) {+ d.why = "the pool has " + std::to_string(threads) + " thread, the split needs 2 or more";+ return d;+ }+ if (rotate) {+ d.why = "STRATA_EXCHANGE_ROTATE=1 moves experts between slots, the split needs fixed slots";+ return d;+ }+ if (!pinned) {+ d.why = "the pool's threads are not pinned";+ return d;+ }+ for (int j = 0; j < threads; ++j) {+ const int c = thread_core[(size_t) j];+ if (c < 0) {+ d.why = j < n_workers ? "worker " + std::to_string(j) + " has no core to be pinned to" : "the host thread has no core";+ return d;+ }+ if ((node[(size_t) j] = node_of(c)) < 0) {+ d.why = "CPU " + std::to_string(c) + " has no known NUMA node";+ return d;+ }+ }+ std::vector<int> nodes = node;+ std::sort(nodes.begin(), nodes.end());+ nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end());+ if (nodes.size() != 2) {+ d.why = "the pool's threads span " + std::to_string(nodes.size()) + " NUMA node(s), the NUMA split supports exactly 2";+ return d;+ }+ d.nodes[0] = nodes[0];+ d.nodes[1] = nodes[1];+ d.slice.assign((size_t) threads, 0);+ for (int j = 0; j < threads; ++j) {+ d.slice[(size_t) j] = node[(size_t) j] == nodes[1] ? 1 : 0;+ ++d.per[d.slice[(size_t) j]];+ }+ if (!numa_slices_balanced(d.per[0], d.per[1])) {+ d.why = "slices too unbalanced (node " + std::to_string(nodes[0]) + ": " + std::to_string(d.per[0]) + " thread" ++ (d.per[0] == 1 ? "" : "s") + ", node " + std::to_string(nodes[1]) + ": " + std::to_string(d.per[1]) + " thread" ++ (d.per[1] == 1 ? "" : "s") + ")";+ d.nodes[0] = d.nodes[1] = -1;+ d.slice.clear();+ d.per[0] = d.per[1] = 0;+ return d;+ }+ d.on = true;+ return d;+}++} // namespace strata::kernels::cpudiff --git a/src/kernels/cpu/numa_pool_test.cpp b/src/kernels/cpu/numa_pool_test.cppnew file mode 100644index 000000000..58092f42b--- /dev/null+++ b/src/kernels/cpu/numa_pool_test.cpp@@ -0,0 +1,146 @@+// numa_pool_test: when the CPU expert pool splits per NUMA node (include/strata/kernels/cpu/numa_pool.hpp), on mock machines.+// No threads, no OS topology: a fake sysfs node folder in a temp directory and a made-up core -> node map. It runs anywhere.+#include "strata/kernels/cpu/numa_pool.hpp"++#include <cstdio>+#include <ctime>+#include <filesystem>+#include <fstream>+#include <string>+#include <vector>++namespace cpu = strata::kernels::cpu;+namespace fs = std::filesystem;++static int g_fail = 0;++static void expect(bool ok, const std::string& what) {+ if (!ok) {+ ++g_fail;+ std::fprintf(stderr, "FAIL: %s\n", what.c_str());+ }+}++// /sys/devices/system/node with `node<i>/cpulist` files; an empty string is a memory-only node+static fs::path make_nodes(const fs::path& root, const std::vector<std::string>& cpulists, bool noise = true) {+ fs::remove_all(root);+ fs::create_directories(root);+ for (size_t i = 0; i < cpulists.size(); ++i) {+ fs::create_directories(root / ("node" + std::to_string(i)));+ std::ofstream(root / ("node" + std::to_string(i)) / "cpulist") << cpulists[i] << "\n";+ }+ if (noise) { // files and folders in that directory that are not nodes+ std::ofstream(root / "online") << "0-1\n";+ std::ofstream(root / "possible") << "0-1\n";+ fs::create_directories(root / "nodes");+ fs::create_directories(root / "node");+ fs::create_directories(root / "nodeX");+ }+ return root;+}++int main() {+ const fs::path tmp = fs::temp_directory_path() / ("strata_numa_pool_test_" + std::to_string((long long) std::time(nullptr)));++ // ---- the node count of a sysfs node folder+ expect(cpu::numa_cpu_node_count_at((tmp / "missing").string()) == 0, "no folder: 0 nodes");+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-23"}).string()) == 1, "one node");+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-11,24-35", "12-23,36-47"}).string()) == 2, "two sockets: 2 nodes");+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-7", "8-15", "16-23", "24-31"}).string()) == 4, "four nodes");+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-23", ""}).string()) == 1, "a memory-only node (CXL) is not a node of CPUs");+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-11", "12-23", ""}).string()) == 2, "two nodes of CPUs and a memory-only one");++ // ---- pick_numa_pool: one node is today's path+ using C = cpu::NumaPoolChoice;+ auto pick = [](const char* pool, const char* split, int nodes) { return cpu::pick_numa_pool(pool, split, nodes); };+ for (int nodes : {0, 1}) {+ for (const char* pool : {(const char*) nullptr, "", "1", "garbage"}) {+ for (const char* split : {(const char*) nullptr, ""}) {+ const auto p = pick(pool, split, nodes);+ expect(p.choice == C::Unchanged && p.log.empty(), "one node, nothing set: Unchanged and silent");+ }+ }+ const auto zero = pick("0", nullptr, nodes);+ expect(zero.choice == C::Unchanged && zero.log.empty(), "one node, STRATA_NUMA_POOL=0: Unchanged and silent");+ expect(pick(nullptr, "1", nodes).choice == C::Unchanged, "one node, STRATA_NUMA_SPLIT=1: the old variable decides");+ expect(pick(nullptr, "test", nodes).choice == C::Unchanged, "one node, STRATA_NUMA_SPLIT=test: the old variable decides");+ }+ {+ const auto p = pick(nullptr, nullptr, 2);+ expect(p.choice == C::Auto && p.log.empty(), "two nodes, nothing set: Auto");+ expect(pick("1", nullptr, 2).choice == C::Auto, "two nodes, STRATA_NUMA_POOL=1: Auto");+ expect(pick("", "", 2).choice == C::Auto, "two nodes, empty variables: Auto");+ expect(pick(nullptr, nullptr, 4).choice == C::Auto, "four nodes: Auto (the decision refuses it, with a line)");+ for (const char* off : {"0", "off", "OFF", "false", "No"}) {+ const auto o = pick(off, nullptr, 2);+ expect(o.choice == C::Off, std::string("two nodes, STRATA_NUMA_POOL=") + off + ": Off");+ expect(o.log.find("STRATA_NUMA_POOL=0") != std::string::npos && o.log.find("2 NUMA nodes") != std::string::npos,+ "the off line names the switch and the node count");+ }+ expect(pick("0", "1", 2).choice == C::Off, "STRATA_NUMA_POOL=0 beats STRATA_NUMA_SPLIT=1");+ expect(pick("0", "test", 1).choice == C::Off, "STRATA_NUMA_POOL=0 beats STRATA_NUMA_SPLIT=test, even on one node");+ expect(pick(nullptr, "1", 2).choice == C::Unchanged, "two nodes, STRATA_NUMA_SPLIT=1: the old variable decides");+ expect(pick(nullptr, "0", 2).choice == C::Unchanged, "two nodes, STRATA_NUMA_SPLIT=0: the old variable decides (off)");+ }++ // ---- decide_numa_split on a made-up 2 x 12-core box (host core 0; cores 0-11 node 0, 12-23 node 1)+ auto node_of = [](int core) { return core < 0 ? -1 : core < 12 ? 0 : 1; };+ auto cores = [](std::initializer_list<int> v) { return std::vector<int>(v); };+ { // 22 workers interleaved over the nodes + the host+ std::vector<int> c;+ for (int i = 0; i < 11; ++i) { c.push_back(1 + i); c.push_back(12 + i); }+ c.push_back(0); // the host+ const auto d = cpu::decide_numa_split(c, 22, true, false, node_of);+ expect(d.on, "22 workers + host on 2 nodes: on");+ expect(d.nodes[0] == 0 && d.nodes[1] == 1, "the nodes are 0 and 1");+ expect(d.per[0] == 12 && d.per[1] == 11, "12 threads on node 0 (the host counted), 11 on node 1");+ expect(d.slice.size() == c.size() && d.slice.back() == 0, "the host drains in slice 0");+ expect(d.slice[0] == 0 && d.slice[1] == 1, "workers alternate between the slices");+ }+ { // all workers on one node: refused+ const auto d = cpu::decide_numa_split(cores({1, 2, 3, 4, 0}), 4, true, false, node_of);+ expect(!d.on && d.why.find("span 1 NUMA node(s)") != std::string::npos, "one node's threads only: refused with the count");+ }+ { // a node with under half of the other's threads: unbalanced+ std::vector<int> c = {1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 0};+ const auto d = cpu::decide_numa_split(c, 10, true, false, node_of);+ expect(!d.on && d.why.find("unbalanced") != std::string::npos, "10 + 1 threads: refused as unbalanced");+ expect(d.nodes[0] == -1 && d.slice.empty() && d.per[0] == 0, "a refused split leaves no plan behind");+ }+ { // three nodes+ auto three = [](int core) { return core < 8 ? 0 : core < 16 ? 1 : 2; };+ const auto d = cpu::decide_numa_split(cores({1, 9, 17, 2, 10, 18, 0}), 6, true, false, three);+ expect(!d.on && d.why.find("span 3 NUMA node(s)") != std::string::npos && d.why.find("exactly 2") != std::string::npos,+ "three nodes: refused, exactly 2 are supported");+ }+ {+ const auto base = cores({1, 12, 2, 13, 0});+ expect(cpu::decide_numa_split(base, 4, true, false, node_of).on, "4 workers + host, balanced: on");+ const auto rot = cpu::decide_numa_split(base, 4, true, true, node_of);+ expect(!rot.on && rot.why.find("STRATA_EXCHANGE_ROTATE=1") != std::string::npos, "rotating slots: refused");+ const auto nopin = cpu::decide_numa_split(base, 4, false, false, node_of);+ expect(!nopin.on && nopin.why.find("not pinned") != std::string::npos, "unpinned threads: refused");+ const auto nocore = cpu::decide_numa_split(cores({1, -1, 2, 13}), 4, true, false, node_of);+ expect(!nocore.on && nocore.why.find("worker 1 has no core") != std::string::npos, "a worker without a core: refused");+ const auto nohost = cpu::decide_numa_split(cores({1, 12, -1}), 2, true, false, node_of);+ expect(!nohost.on && nohost.why.find("host thread has no core") != std::string::npos, "a host without a core: refused");+ auto unknown = [](int core) { return core == 12 ? -1 : 0; };+ const auto un = cpu::decide_numa_split(base, 4, true, false, unknown);+ expect(!un.on && un.why.find("CPU 12 has no known NUMA node") != std::string::npos, "a core with no node: refused");+ const auto one = cpu::decide_numa_split(cores({1}), 1, true, false, node_of);+ expect(!one.on && one.why.find("needs 2 or more") != std::string::npos, "one thread: refused");+ }+ { // a single-node machine fed to the decision (STRATA_NUMA_SPLIT=1 on one node): refused, never half-on+ auto flat = [](int core) { return core < 0 ? -1 : 0; };+ const auto d = cpu::decide_numa_split(cores({1, 2, 3, 4, 0}), 4, true, false, flat);+ expect(!d.on, "every core on node 0: off");+ }++ fs::remove_all(tmp);+ if (g_fail) {+ std::fprintf(stderr, "numa_pool_test: %d failure(s)\n", g_fail);+ return 1;+ }+ std::printf("numa_pool_test: OK\n");+ return 0;+}diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cppindex d2a91fb9d..e3f113d33 100644--- a/src/kernels/cpu/pool.cpp+++ b/src/kernels/cpu/pool.cpp@@ -3,6 +3,7 @@#include "strata/core/progress.hpp"#include "strata/platform/aux_cpus.hpp"#include "strata/kernels/cpu/expert_layout.hpp"+#include "strata/kernels/cpu/numa_pool.hpp"#include <algorithm>#include <atomic>@@ -544,6 +545,34 @@ int cpu_numa_node(int cpu) {}#endif+// The NUMA nodes with CPUs this machine has (0.2.2: the split is tried by itself from 2 on). Linux: the sysfs node folder+// (numa_pool.hpp); Windows: GetNumaHighestNodeNumber; any other system, or a sandbox that hides the folder: 1.+int numa_node_count() {+#if defined(__linux__)+ return (std::max)(1, numa_cpu_node_count_at("/sys/devices/system/node"));+#elif defined(_WIN32)+ ULONG highest = 0;+ return GetNumaHighestNodeNumber(&highest) ? (int) highest + 1 : 1;+#else+ return 1;+#endif+}++// The real-node split for threads on `thread_core` (numa_pool.hpp). Linux only: the placement of the experts' memory+// (first touch by threads pinned to a node, move_pages to check) has no Windows or macOS version.+NumaSplitDecision decide_real_numa_split([[maybe_unused]] const std::vector<int>& thread_core, [[maybe_unused]] int n_workers,+ [[maybe_unused]] bool pinned) {+#if defined(__linux__)+ // the expert source's own reading of it (expert_source.cpp): exactly "1"+ const char* rotate = std::getenv("STRATA_EXCHANGE_ROTATE");+ return decide_numa_split(thread_core, n_workers, pinned, rotate != nullptr && std::strcmp(rotate, "1") == 0, cpu_numa_node);+#else+ NumaSplitDecision d;+ d.why = "the split needs Linux";+ return d;+#endif+}+} // namespaceThreadAffinity pin_current_thread(int core) {@@ -650,37 +679,77 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity afif (tasks < 0 || tasks > kMaxTasks) throw std::invalid_argument("expert pool tasks must be in 0..4096");if (const char* e = std::getenv("STRATA_POOL_SPIN_US")) // a test knob; see kSpinBeforeSleepspin_before_sleep_ = std::chrono::microseconds((std::max)(0, std::atoi(e)));- // STRATA_NUMA_SPLIT, read here for the worker count (the split itself is decided below, once the count is known)+ // STRATA_NUMA_SPLIT, read here for the worker count (the split itself is decided below, once the count is known).+ // 0.2.2: with 2 or more NUMA nodes and neither variable set the split is tried by itself, and STRATA_NUMA_POOL=0 turns that+ // off (numa_pool.hpp). One node: pick_numa_pool says Unchanged and nothing below differs from the pool before the split.const char* env = std::getenv("STRATA_NUMA_SPLIT");- const bool env_off = env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0;- const bool env_known = !env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0);+ const int numa_nodes = numa_node_count();+ const NumaPoolPick pool_pick = pick_numa_pool(std::getenv("STRATA_NUMA_POOL"), env, numa_nodes);+ if (!pool_pick.log.empty()) std::fprintf(stderr, "%s\n", pool_pick.log.c_str());+ if (pool_pick.choice == NumaPoolChoice::Off) env = nullptr;+ bool auto_pool = pool_pick.choice == NumaPoolChoice::Auto; // the split without STRATA_NUMA_SPLIT: only if it can run+ bool env_off = !auto_pool && (env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0);+ bool env_known = auto_pool || (!env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0));const PoolPlanInput plan_in = pool_plan_input(topo_);const PoolPlan wplan = plan_pool_workers(plan_in);- // the split needs workers on both nodes: its cores are taken round-robin over the packages, so any count under- // the core count (the default or a configured one) spans them evenly (one package: the order is kept)- if (env_known && topo_.worker_package.size() == topo_.worker_cores.size()) {+ // the split needs workers on both nodes: its cores are taken round-robin over the nodes (the packages when a core's node+ // is unknown), so any count under the core count (the default or a configured one) spans them evenly (one node: the order+ // is kept)+ auto interleave_cores = [&] {+ if (topo_.worker_package.size() != topo_.worker_cores.size()) return;+ std::vector<int> key = topo_.worker_package;+#if defined(__linux__)+ std::vector<int> by_node;+ for (int c : topo_.worker_cores) by_node.push_back(cpu_numa_node(c));+ if (std::none_of(by_node.begin(), by_node.end(), [](int n) { return n < 0; })) key = by_node;+#endifstd::vector<int> cores, pkgs;- for (size_t i : interleave_packages(topo_.worker_package)) {+ for (size_t i : interleave_packages(key)) {cores.push_back(topo_.worker_cores[i]);pkgs.push_back(topo_.worker_package[i]);}topo_.worker_cores = std::move(cores);topo_.worker_package = std::move(pkgs);- }- if (n_workers > 0) {- n_ = n_workers;- } else if (topo_.is_hybrid && affinity_ != PoolAffinity::All) {- n_ = (std::max)(1, topo_.p_cores - 1);+ };+ // the worker count: configured, else the hybrid P cores, else the plan (`span`: every socket counts, the split's default)+ auto pick_workers = [&](bool span, const char** limit) -> int {+ if (n_workers > 0) return n_workers;+ if (topo_.is_hybrid && affinity_ != PoolAffinity::All) return (std::max)(1, topo_.p_cores - 1);+ if (span) {+ PoolPlanInput span_in = plan_in;+ span_in.span_packages = true;+ const PoolPlan sp = plan_pool_workers(span_in);+ *limit = sp.limit;+ return (std::min)(sp.workers, (std::max)(1, (int) topo_.worker_cores.size()));+ }+ *limit = wplan.limit;+ return (std::min)(wplan.workers, (std::max)(1, (int) topo_.worker_cores.size()));+ };+ if (auto_pool) {+ // try the split on a copy: when it cannot run the cores, the count and the plan stay exactly what they were+ const CpuTopology plain_topo = topo_;+ interleave_cores();+ const char* try_limit = nullptr;+ const int try_n = pick_workers(true, &try_limit);+ std::vector<int> thread_core;+ for (int j = 0; j < try_n; ++j)+ thread_core.push_back(pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1);+ if (host_works_) thread_core.push_back(topo_.host_core);+ const NumaSplitDecision tried = decide_real_numa_split(thread_core, try_n, pin);+ if (!tried.on) {+ topo_ = plain_topo;+ auto_pool = env_known = false;+ env_off = true;+ std::fprintf(stderr, "strata cpu pool: NUMA pool off: %s (%d NUMA nodes found), the pool runs as before\n",+ tried.why.c_str(), numa_nodes);+ }} else if (env_known) {- // ... and the default spans every socket (the plan's other bounds stay)- PoolPlanInput span_in = plan_in;- span_in.span_packages = true;- const PoolPlan sp = plan_pool_workers(span_in);- n_ = (std::min)(sp.workers, (std::max)(1, (int) topo_.worker_cores.size()));- plan_limit_ = sp.limit;- } else {- n_ = (std::min)(wplan.workers, (std::max)(1, (int) topo_.worker_cores.size()));- plan_limit_ = wplan.limit;+ interleave_cores();+ }+ {+ const char* limit = nullptr;+ n_ = pick_workers(env_known, &limit);+ if (limit != nullptr) plan_limit_ = limit;}plan_cores_ = wplan.cores;plan_logical_ = plan_in.logical;@@ -700,58 +769,27 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity afplan.why = std::string("unknown STRATA_NUMA_SPLIT value '") + env + "' (use 1 or test)";std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());} else if (env_known) {- plan.test = std::strcmp(env, "test") == 0;+ plan.test = !auto_pool && std::strcmp(env, "test") == 0;const int threads = n_ + (host_works_ ? 1 : 0); // thread j < n_: worker j; j == n_: the host- [[maybe_unused]] auto core_of = [&](int j) {- if (j == n_) return topo_.host_core;- return pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1;- };std::vector<int> slice((size_t) threads, 0);- char why[160] = "";+ std::string why;if (threads < 2) {- std::snprintf(why, sizeof why, "the pool has %d thread, the split needs 2 or more", threads);+ why = "the pool has " + std::to_string(threads) + " thread, the split needs 2 or more";} else if (plan.test) {for (int j = 0; j < threads; ++j) slice[(size_t) j] = j % 2;} else {-#if defined(__linux__)- std::vector<int> node((size_t) threads, -1);- // the expert source's own reading of it (expert_source.cpp): exactly "1"- const char* rotate = std::getenv("STRATA_EXCHANGE_ROTATE");- if (rotate != nullptr && std::strcmp(rotate, "1") == 0)- std::snprintf(why, sizeof why, "STRATA_EXCHANGE_ROTATE=1 moves experts between slots, the split needs "- "fixed slots");- else if (!pin) std::snprintf(why, sizeof why, "the pool's threads are not pinned");- for (int j = 0; j < threads && why[0] == '\0'; ++j) {- const int c = core_of(j);- if (c < 0 && j < n_) std::snprintf(why, sizeof why, "worker %d has no core to be pinned to", j);- else if (c < 0) std::snprintf(why, sizeof why, "the host thread has no core");- else if ((node[(size_t) j] = cpu_numa_node(c)) < 0)- std::snprintf(why, sizeof why, "CPU %d has no known NUMA node", c);+ std::vector<int> thread_core;+ for (int j = 0; j < threads; ++j)+ thread_core.push_back(j == n_ ? topo_.host_core+ : pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1);+ const NumaSplitDecision d = decide_real_numa_split(thread_core, n_, pin);+ why = d.why;+ if (d.on) {+ slice = d.slice;+ plan.nodes[0] = d.nodes[0];+ plan.nodes[1] = d.nodes[1];+ for (int j = 0; j < threads; ++j) plan.cores[slice[(size_t) j]].push_back(thread_core[(size_t) j]);}- if (why[0] == '\0') {- std::vector<int> nodes = node;- std::sort(nodes.begin(), nodes.end());- nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end());- if (nodes.size() != 2) {- std::snprintf(why, sizeof why, "the pool's threads span %zu NUMA node(s), the NUMA split supports exactly 2",- nodes.size());- } else {- plan.nodes[0] = nodes[0];- plan.nodes[1] = nodes[1];- int per[2] = {0, 0};- for (int j = 0; j < threads; ++j) {- slice[(size_t) j] = node[(size_t) j] == nodes[1] ? 1 : 0;- plan.cores[slice[(size_t) j]].push_back(core_of(j));- ++per[slice[(size_t) j]];- }- if (!numa_slices_balanced(per[0], per[1]))- std::snprintf(why, sizeof why, "slices too unbalanced (node %d: %d thread%s, node %d: %d thread%s)",- nodes[0], per[0], per[0] == 1 ? "" : "s", nodes[1], per[1], per[1] == 1 ? "" : "s");- }- }-#else- std::snprintf(why, sizeof why, "the split needs Linux");-#endif}plan.why = why;plan.on = plan.why.empty();@@ -767,6 +805,9 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity afhslice_ = host_works_ ? slice[(size_t) n_] : 0;slice_threads_.assign(2, 0);for (int k : slice) ++slice_threads_[(size_t) k];+ if (auto_pool)+ std::fprintf(stderr, "strata cpu pool: NUMA pool on: %d NUMA nodes found, one slice of the pool per node "+ "(STRATA_NUMA_POOL=0 turns it off)\n", numa_nodes);if (plan.test)std::fprintf(stderr, "strata cpu pool: NUMA split on (test, no real nodes): slice 0 = %d threads, ""slice 1 = %d threads\n", slice_threads_[0], slice_threads_[1]);