Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
fix: refuse a lopsided NUMA split and accept only 1/test for STRATA_NUMA_SPLIT
Baptiste Rajaut2496795bd94c
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hppindex fb63a1cbc..600295844 100644--- a/include/strata/core/expert_source.hpp+++ b/include/strata/core/expert_source.hpp@@ -121,7 +121,8 @@ uint64_t numa_pin_free(uint64_t global_free, const std::vector<uint64_t>& node_f/// its slice-1 bytes the rest. Page rule (absolute arena offsets): any page holding a slice-0 byte belongs to slice 0,/// so slice-0 ranges round OUTWARDS (start down, end up) and slice-1 ranges round INWARDS (start up, end down) - this/// also settles pages shared across role and slot boundaries. Returns three ranges in arena offsets, possibly empty-/// (first == second).+/// (first == second). Adjacent roles smaller than a page may give overlapping slice-0 ranges: harmless (same node, the+/// caller coalesces them).std::array<std::pair<uint64_t, uint64_t>, 3> numa_slice_ranges(uint64_t slot_off, const uint64_t role_bytes[3],int slice, uint64_t page = 4096);/// The sentence appended to a "not enough RAM" start failure: how much of the RAM is file cache (Cached + Buffers ofdiff --git a/src/core/expert_source.cpp b/src/core/expert_source.cppindex 4c4ae82fd..26c7bfe4d 100644--- a/src/core/expert_source.cpp+++ b/src/core/expert_source.cpp@@ -21,6 +21,7 @@#include <algorithm>#include <atomic>+#include <cassert>#include <cerrno>#include <chrono>#include <cstdio>@@ -130,6 +131,7 @@ uint64_t numa_pin_free(uint64_t global_free, const std::vector<uint64_t>& node_fstd::array<std::pair<uint64_t, uint64_t>, 3> numa_slice_ranges(uint64_t slot_off, const uint64_t role_bytes[3],int slice, uint64_t page) {+ assert(slice == 0 || slice == 1);std::array<std::pair<uint64_t, uint64_t>, 3> out{};uint64_t start = slot_off;for (int r = 0; r < 3; ++r) {@@ -538,6 +540,8 @@ struct NumaComplementPlacement {std::chrono::steady_clock::time_point t0;static std::vector<uint64_t> node_free() {+ // CBA: stops at the first missing node directory (as free_memory_bytes does), sparse node ids give a false+ // "at risk 0.00" line only; list the node directory if a sparse-id host ever shows itstd::vector<uint64_t> out;for (int n = 0; n < 64; ++n) {std::ifstream node("/sys/devices/system/node/node" + std::to_string(n) + "/meminfo");@@ -567,6 +571,7 @@ struct NumaComplementPlacement {/// Decides from the live pool's plan; false with a logged reason when the split is on but placement cannot run.void init(const std::vector<uint64_t>& offsets, int64_t n_expert, const std::vector<uint64_t>& blob_bytes,const std::vector<uint64_t>& role_bytes) {+ // relies on the pool being built first (generate.cpp: the ExpertPool at ~5632, the complement at ~7690)const auto& plan = strata::kernels::cpu::active_numa_split();if (!plan.on || plan.test) return;free_before = node_free();@@ -580,7 +585,7 @@ struct NumaComplementPlacement {else if (plan.nodes[k] < 0 || plan.nodes[k] >= 64) why = "a node id is out of range";if (why != nullptr) {std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement FAILED: %s; the complement is placed by "- "first touch as usual (this arm is invalid)\n", why);+ "first touch as usual (the split runs with remote reads)\n", why);return;}for (int k = 0; k < 2; ++k) {@@ -625,6 +630,7 @@ struct NumaComplementPlacement {for (uint64_t at = std::max(it->first, lo); at < std::min(it->second, hi); at += 4096)((volatile uint8_t*) arena)[at] = 0;};+ // CBA: a failed spawn terminates; fall back to an unpinned touch if it ever happensstd::thread a(touch, 0), b(touch, 1);a.join();b.join();@@ -654,7 +660,7 @@ struct NumaComplementPlacement {/// Samples one page in 64 of each slice with move_pages (null node list: returns each page's node) and logs the/// share on the intended node, once the arena is registered (its final state); `got < bytes` means the paced loop- /// stopped early, `registered < bytes` that the driver did not page-lock all of it: the arm then fails.+ /// stopped early, `registered < bytes` that the driver did not page-lock all of it: the placement then fails.void report(const uint8_t* arena, uint64_t got, uint64_t registered, uint64_t bytes, const char* stop) const {const double secs = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();uint64_t on_node[2] = {0, 0}, sampled[2] = {0, 0}, per_node[2][64] = {}, absent[2] = {0, 0};@@ -682,17 +688,17 @@ struct NumaComplementPlacement {double share[2];for (int k = 0; k < 2; ++k) share[k] = sampled[k] > 0 ? 100.0 * (double) on_node[k] / (double) sampled[k] : 0.0;std::fprintf(stderr, "FileExpertSource: NUMA placement: node %d %.2f GiB (%.1f %% of its slices), node %d %.2f GiB "- "(%.1f %% of its slices); free before: %s; after the drop: %s; pages taken and page-locked in %.1f s\n",+ "(%.1f %% of its slices); free before: %s; after the drop: %s; pages taken and page-locked in %.1f s since the placement started\n",nodes[0], share[0] / 100.0 * (double) slice_bytes[0] / 1073741824.0, share[0], nodes[1],share[1] / 100.0 * (double) slice_bytes[1] / 1073741824.0, share[1], free_text(free_before).c_str(),free_text(free_after).c_str(), secs);if (got < bytes)std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement FAILED: the paced loop stopped at %.2f of %.2f "- "GiB (%s); the rest is first-touched by unpinned copy threads (this arm is invalid)\n",+ "GiB (%s); the rest is first-touched by unpinned copy threads (memory not placed on its node: the split runs with remote reads)\n",(double) got / 1073741824.0, (double) bytes / 1073741824.0, stop != nullptr ? stop : "?");else if (registered < bytes)std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement FAILED: only %.2f of %.2f GiB page-locked (%s); "- "the unregistered pages can be migrated by NUMA balancing (this arm is invalid)\n",+ "the unregistered pages can be migrated by NUMA balancing (memory not placed on its node: the split runs with remote reads)\n",(double) registered / 1073741824.0, (double) bytes / 1073741824.0, stop != nullptr ? stop : "?");if (pin_failed)std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement: a toucher could not be pinned to its core\n");@@ -3740,15 +3746,6 @@ bool FileExpertSource::pin_cache_complement(std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement FAILED: the complement is not page-locked, ""nothing placed\n");}- // STRATA_COMPLEMENT_NOHUGEPAGE=1 (a test knob: the TLB cost of 4 KiB pages without the split) acts on the paced- // arena only- static const bool nohuge = [] {- const char* v = std::getenv("STRATA_COMPLEMENT_NOHUGEPAGE");- return v != nullptr && std::strcmp(v, "1") == 0;- }();- if (nohuge && !paced)- std::fprintf(stderr, "FileExpertSource: STRATA_COMPLEMENT_NOHUGEPAGE=1 ignored: the complement was not "- "allocated in steps\n");#endifif (pin && !paced) {const cudaError_t allocated = cudaHostAlloc(&arena, (size_t) bytes,@@ -3789,14 +3786,11 @@ bool FileExpertSource::pin_cache_complement(#endif#if defined(__linux__)// STRATA_NUMA_SPLIT: a 2 MiB huge page would put several slices on its toucher's node, so the arena- // takes 4 KiB pages (one call, no VMA split); STRATA_COMPLEMENT_NOHUGEPAGE=1 does this alone- if (arena != nullptr && (numa.place || nohuge)) {+ // takes 4 KiB pages (one call, no VMA split)+ if (arena != nullptr && numa.place) {if (madvise(arena, (size_t) ((bytes + page - 1) / page * page), MADV_NOHUGEPAGE) != 0)std::fprintf(stderr, "FileExpertSource: WARNING: madvise(MADV_NOHUGEPAGE) on the complement ""failed (errno %d)\n", errno);- else if (!numa.place)- std::fprintf(stderr, "FileExpertSource: STRATA_COMPLEMENT_NOHUGEPAGE=1: the complement takes "- "4 KiB pages\n");}#endif} else {diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cppindex 58bdd7849..fc77f9ce4 100644--- a/src/kernels/cpu/pool.cpp+++ b/src/kernels/cpu/pool.cpp@@ -613,6 +613,8 @@ void ExpertPool::diag(std::FILE* f) const {const uint64_t h = head_.load();uint32_t batch = (uint32_t) (h >> 32), claimed = (uint32_t) h & 0xffffu, total = (uint32_t) (h >> 16) & 0xffffu,done = done_.load();+ // CBA: this summed-slices branch is never run by a test (it needs a diag call mid-phase), it is a trivial sum, a mid-phase+ // diag test if it ever misreportsif (numa_n_ > 0 && mode_ >= 5) { // a split native batch never writes head_/done_: the slices' sums insteadclaimed = total = done = 0;for (int k = 0; k < numa_n_; ++k) {@@ -676,7 +678,13 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af// STRATA_NUMA_SPLIT: decided here, before any worker starts (a worker counts into its slice's `parked` from its// first instruction), and never revised. Unset: the plan stays empty and nothing below runs.NumaSplitPlan plan;- if (const char* env = std::getenv("STRATA_NUMA_SPLIT"); env != nullptr && *env != '\0' && std::strcmp(env, "0") != 0) {+ const char* env = std::getenv("STRATA_NUMA_SPLIT");+ const bool env_off = env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0;+ const bool env_known = !env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0);+ if (!env_off && !env_known) {+ plan.why = std::string("unknown STRATA_NUMA_SPLIT value '") + env + "' (use 1 or test)";+ std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());+ } else if (env_known) {plan.test = std::strcmp(env, "test") == 0;const int threads = n_ + (host_works_ ? 1 : 0); // thread j < n_: worker j; j == n_: the host[[maybe_unused]] auto core_of = [&](int j) {@@ -710,15 +718,21 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity afstd::sort(nodes.begin(), nodes.end());nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end());if (nodes.size() != 2) {- std::snprintf(why, sizeof why, "the pool's threads span %zu NUMA node(s), the prototype needs exactly 2",+ std::snprintf(why, sizeof why, "the pool's threads span %zu NUMA node(s), the NUMA split supports exactly 2",nodes.size());} else {plan.nodes[0] = nodes[0];plan.nodes[1] = nodes[1];+ int per[2] = {0, 0};for (int j = 0; j < threads; ++j) {slice[(size_t) j] = node[(size_t) j] == nodes[1] ? 1 : 0;plan.cores[slice[(size_t) j]].push_back(core_of(j));+ ++per[slice[(size_t) j]];}+ // rows stay 50/50, so a slice with under half the other's threads would be the straggler of every batch+ if (std::min(per[0], per[1]) * 2 < std::max(per[0], per[1]))+ std::snprintf(why, sizeof why, "slices too unbalanced (node %d: %d thread%s, node %d: %d thread%s)",+ nodes[0], per[0], per[0] == 1 ? "" : "s", nodes[1], per[1], per[1] == 1 ? "" : "s");}}#elsediff --git a/src/kernels/cpu/pool_tasks_test.cpp b/src/kernels/cpu/pool_tasks_test.cppindex 17b5341a3..d04d8fca2 100644--- a/src/kernels/cpu/pool_tasks_test.cpp+++ b/src/kernels/cpu/pool_tasks_test.cpp@@ -227,6 +227,30 @@ int main(int argc, char** argv) {unset_env("STRATA_NUMA_SPLIT");if (!refused) { std::fprintf(stderr, "numa-split did not refuse STRATA_EXCHANGE_ROTATE=1\n"); return 1; }}+ { // the real split on any host: on means two distinct nodes and cores on both slices, off always says why+ put_env("STRATA_NUMA_SPLIT", "1");+ bool ok = false;+ {+ cpu::ExpertPool pool(4, true, true);+ const auto& plan = cpu::active_numa_split();+ ok = plan.on ? plan.nodes[0] != plan.nodes[1] && !plan.cores[0].empty() && !plan.cores[1].empty()+ : !plan.why.empty();+ std::printf("pool tasks: real split %s%s\n", plan.on ? "on" : "off: ", plan.why.c_str());+ }+ unset_env("STRATA_NUMA_SPLIT");+ if (!ok) { std::fprintf(stderr, "numa-split real-mode plan invariant broken\n"); return 1; }+ // a value other than 1 / test is off, with the value in the reason; "0" and "" are off silently+ for (const char* v : {"off", "yes"}) {+ put_env("STRATA_NUMA_SPLIT", v);+ bool bad = false;+ {+ cpu::ExpertPool pool(2, true, true);+ bad = cpu::active_numa_split().on || cpu::active_numa_split().why.find(std::string("'") + v + "'") == std::string::npos;+ }+ unset_env("STRATA_NUMA_SPLIT");+ if (bad) { std::fprintf(stderr, "numa-split did not refuse STRATA_NUMA_SPLIT=%s\n", v); return 1; }+ }+ }#endifstd::printf("pool tasks: %d bitwise comparisons passed, %d of them in NUMA-split mode (%d with modes 0-4 ""interleaved)\n", checks + split_checks + mixed_checks, split_checks + mixed_checks, mixed_checks);diff --git a/src/program/generate.cpp b/src/program/generate.cppindex ea7cb608d..427957b9d 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -5824,6 +5824,9 @@ int main(int argc, char** argv) {if (pool.plan_limit()[0] != '\0') // the default count: what it was bounded by (a configured count is never changed)std::fprintf(stderr, "strata generate: CPU pool: %d workers by default (%d physical cores, %d logical processors; bounded by %s)\n",pool.workers(), pool.plan_cores(), pool.plan_logical(), pool.plan_limit());+ if (srcp == &arena_src && strata::kernels::cpu::active_numa_split().on && !strata::kernels::cpu::active_numa_split().test)+ std::fprintf(stderr, "strata generate: NUMA split on, but this model's experts come from the arena path: memory is "+ "not placed per node (known limitation)\n");if (pool.is_hybrid() && pool.affinity() != strata::kernels::cpu::PoolAffinity::All) {const char* aff_str = pool.affinity() == strata::kernels::cpu::PoolAffinity::PCores ? "p-cores" :pool.affinity() == strata::kernels::cpu::PoolAffinity::All ? "all" : "auto";