Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
feat: per-NUMA-node task slices and counters in the CPU expert pool (opt-in)
Baptiste Rajaut709f8dc63d5f
diff --git a/include/strata/kernels/cpu/pool.hpp b/include/strata/kernels/cpu/pool.hppindex 36e8c89cc..9dcb349bd 100644--- a/include/strata/kernels/cpu/pool.hpp+++ b/include/strata/kernels/cpu/pool.hpp@@ -38,6 +38,7 @@#include <condition_variable>#include <cstdint>#include <mutex>+#include <string>#include <thread>#include <vector>@@ -153,6 +154,20 @@ struct ThreadAffinity {ThreadAffinity pin_current_thread(int core);void restore_thread_affinity(const ThreadAffinity& previous);+/// STRATA_NUMA_SPLIT's decision for a pool (see `ExpertPool`'s NodeSync). `1` splits on the real NUMA nodes of+/// the pinned threads (Linux, exactly 2 nodes); `test` splits by thread index (thread j -> slice j % 2) with no real+/// node, for the parity test.+struct NumaSplitPlan {+ bool on = false; ///< split enabled+ bool test = false; ///< STRATA_NUMA_SPLIT=test: no real nodes, never places memory+ int nodes[2] = {-1, -1}; ///< real node id of slice 0 and 1 (ascending); -1 in test mode+ std::vector<int> cores[2]; ///< the cores the pool pinned for each slice (workers + host)+ std::string why; ///< reason when off ("" when on, or when the env var is unset)+};+/// The live pool's decision (set by the ExpertPool constructor, reset by its destructor); `on == false` when no+/// pool exists or the split is off.+const NumaSplitPlan& active_numa_split();+class ExpertPool {public:/// `n_workers <= 0` means "every physical core except the first". Workers are pinned to physical cores@@ -185,6 +200,10 @@ public:const char* plan_limit() const { return plan_limit_; }int plan_cores() const { return plan_cores_; }int plan_logical() const { return plan_logical_; }+ /// STRATA_NUMA_SPLIT: 2 when the native row phases are split per NUMA node, 0 when off.+ int numa_slices() const { return numa_n_; }+ /// Accumulated drain time of slice `s` over the mode-5/6 phases, ms: publish to the slice's last `done`.+ double numa_slice_ms(int s) const { return s >= 0 && s < numa_n_ ? slice_ms_[s] : 0.0; }/// Whether the host thread also drains. Reported at startup, because "the engine adapts to the machine it/// is on" is only true if the engine says which adaptation it took.bool host_works() const { return host_works_; }@@ -326,6 +345,48 @@ private:};const NativeFmt* nfmt_ = nullptr;std::vector<SplitBufMulti> split_multi_;+ // STRATA_NUMA_SPLIT=1 (opt-in, Linux, exactly 2 nodes): in the native row phases (modes 5/6) every expert's rows+ // are cut into one slice per NUMA node (slice k = rows [per*k/2, per*(k+1)/2) of the gate/up or down matrix, the+ // part the resident complement places on node k in step B), and each node's threads claim and count on their own+ // slice's counters; in every mode they park on their own `parked`. Off (`numa_n_ == 0`, the decision is final in+ // the constructor): none of this is touched and the pool is as before.+ //+ // THE CLAIM PROTOCOL IS issue #29's, ONCE PER SLICE. Each slice head packs `epoch | njobs | index` and is claimed+ // with the same CAS as `head_`. A mode-5/6 batch stores every slice head with its epoch and leaves `head_` alone;+ // any other batch stores `head_` and leaves the slice heads alone. So for batch `e` exactly one family of heads+ // carries `e`, and a stale head is always exhausted (`wait_done` saw every task of it done). A worker therefore+ // never reads `mode_` to pick a counter: it tries its slice head, then `head_`, and the epoch tag makes the wrong+ // one fail - as it makes a late worker's claim fail.+ struct NodeSync {+ alignas(64) std::atomic<uint64_t> head{0}; // as `head_`, for this slice of the phase+ alignas(64) std::atomic<uint32_t> done{0}; // as `done_`+ alignas(64) std::atomic<uint32_t> parked{0}; // as `parked_`, this slice's workers only+ alignas(64) std::atomic<int64_t> drained_ns{0}; // steady_clock ns when `done` reached `tasks` (0: not yet)+ // the batch's description, written by the host before the head's release store, read after a claim+ alignas(64) int base = 0; // the slice's first task index in the phase (task ids are global for `diag`)+ int tasks = 0; // T: the slice's task count+ int h = 0; // rows of each expert in the slice+ int row0 = 0; // the slice's first matrix row (per*k/2)+ int64_t rows = 0; // R = experts * h+ };+ // CBA: the NodeSync blocks live wherever `new` puts them (the host's node); per-node pages if the remote home+ // of a line ever shows up in the counters.+ std::unique_ptr<NodeSync[]> nsync_;+ int numa_n_ = 0; // slices (2 when on), 0 = off+ std::vector<int> wslice_; // each worker's slice+ int hslice_ = 0; // the host's slice (when it drains)+ std::vector<int> slice_threads_; // threads draining each slice (the host counted in its own)+ int64_t pub_ns_ = 0; // the current mode-5/6 phase's publish time (host only)+ double slice_ms_[2] = {0.0, 0.0}; // accumulated per-slice drain time (host only)+ bool numa_phase() const { return numa_n_ > 0 && mode_ >= 5; }+ /// The claim of `claim()` on slice `slice`'s head: `base + i`, or -1.+ int claim_slice(uint32_t epoch, int slice);+ std::atomic<uint32_t>& parked_of(int id) { return numa_n_ > 0 ? nsync_[(size_t) wslice_[(size_t) id]].parked : parked_; }+ uint32_t parked_count() const;+ /// Fill the slices' descriptions for a mode-5/6 phase of `nb` experts of `per` rows; returns the total tasks.+ int numa_plan_phase(int per, int nb);+ /// Per-slice counters for the stall messages and `diag` ("" when off).+ std::string numa_counters() const;PoolAffinity affinity_ = PoolAffinity::All;CpuTopology topo_;};diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cppindex 80161b9a2..772a68781 100644--- a/src/kernels/cpu/pool.cpp+++ b/src/kernels/cpu/pool.cpp@@ -27,6 +27,7 @@#else#include <pthread.h>#include <sched.h>+#include <unistd.h>#include "pool_affinity_linux.hpp"#endif@@ -530,6 +531,19 @@ bool pin_this_thread(int core, [[maybe_unused]] int worker = -1) {#endif}+#if defined(__linux__)+// The NUMA node of a logical CPU (sysfs), or -1.+int cpu_numa_node(int cpu) {+ if (cpu < 0) return -1;+ char path[96];+ for (int node = 0; node < 64; ++node) {+ std::snprintf(path, sizeof path, "/sys/devices/system/cpu/cpu%d/node%d", cpu, node);+ if (access(path, F_OK) == 0) return node;+ }+ return -1;+}+#endif+} // namespaceThreadAffinity pin_current_thread(int core) {@@ -583,13 +597,24 @@ int64_t now_ms() {return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();}+int64_t now_ns() {+ return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch())+ .count();+}+// CBA: one plain global, written by the constructor and destructor of the pool that owns it; fine while pools are+// built and destroyed on one thread (generate's measure pools, then the main one), an atomic owner if that changes.+NumaSplitPlan g_numa_plan;+const ExpertPool* g_numa_owner = nullptr;} // namespace+const NumaSplitPlan& active_numa_split() { return g_numa_plan; }+void ExpertPool::diag(std::FILE* f) const {const uint64_t h = head_.load();std::fprintf(f, " expert pool: epoch %u, batch epoch %u: %u of %u jobs claimed, %u done; %u of %d workers parked, ""%u sleeping; mode %d\n", epoch_.load(), (uint32_t) (h >> 32), (uint32_t) h & 0xffffu,- (uint32_t) (h >> 16) & 0xffffu, done_.load(), parked_.load(), n_, sleepers_.load(), mode_);+ (uint32_t) (h >> 16) & 0xffffu, done_.load(), parked_count(), n_, sleepers_.load(), mode_);+ if (numa_n_ > 0) std::fprintf(f, " expert pool NUMA slices:%s\n", numa_counters().c_str());std::fprintf(f, " expert pool threads:");for (int i = 0; i < n_; ++i) {const int32_t s = wstate_[(size_t) i].load();@@ -612,20 +637,20 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity afif (const char* e = std::getenv("STRATA_POOL_SPIN_US")) // a test knob; see kSpinBeforeSleepspin_before_sleep_ = std::chrono::microseconds((std::max)(0, std::atoi(e)));const PoolPlanInput plan_in = pool_plan_input(topo_);- const PoolPlan plan = plan_pool_workers(plan_in);+ const PoolPlan wplan = plan_pool_workers(plan_in);if (n_workers > 0) {n_ = n_workers;// recommend, never force: a count above what this CPU uses well runs as configured, with a plain warningstatic std::atomic<bool> warned{false};- const std::string why = pool_workers_warning(n_workers, plan_in, plan);+ const std::string why = pool_workers_warning(n_workers, plan_in, wplan);if (!why.empty() && !warned.exchange(true)) std::fprintf(stderr, "strata cpu pool: warning: %s\n", why.c_str());} else if (topo_.is_hybrid && affinity_ != PoolAffinity::All) {n_ = (std::max)(1, topo_.p_cores - 1);} else {- n_ = (std::min)(plan.workers, (std::max)(1, (int) topo_.worker_cores.size()));- plan_limit_ = plan.limit;+ n_ = (std::min)(wplan.workers, (std::max)(1, (int) topo_.worker_cores.size()));+ plan_limit_ = wplan.limit;}- plan_cores_ = plan.cores;+ plan_cores_ = wplan.cores;plan_logical_ = plan_in.logical;if (n_ < 1) n_ = 1;scratch_.resize((size_t) n_);@@ -636,12 +661,83 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity afstrata::core::diag_pool_fn().store(&diag_active_pool);split_.resize((size_t) kMaxSplit);split_multi_.resize((size_t) kMaxSplitMulti);+ // STRATA_NUMA_SPLIT: decided here, before any worker starts (a worker counts into its slice's `parked` from its+ // first instruction), and never revised. Unset: the plan stays empty and nothing below runs.+ NumaSplitPlan plan;+ if (const char* env = std::getenv("STRATA_NUMA_SPLIT"); env != nullptr && *env != '\0' && std::strcmp(env, "0") != 0) {+ plan.test = std::strcmp(env, "test") == 0;+ const int threads = n_ + (host_works_ ? 1 : 0); // thread j < n_: worker j; j == n_: the host+ [[maybe_unused]] auto core_of = [&](int j) {+ if (j == n_) return topo_.host_core;+ return pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1;+ };+ std::vector<int> slice((size_t) threads, 0);+ char why[160] = "";+ if (threads < 2) {+ std::snprintf(why, sizeof why, "the pool has %d thread, the split needs 2 or more", threads);+ } else if (plan.test) {+ for (int j = 0; j < threads; ++j) slice[(size_t) j] = j % 2;+ } else {+#if defined(__linux__)+ std::vector<int> node((size_t) threads, -1);+ if (!pin) std::snprintf(why, sizeof why, "the pool's threads are not pinned");+ for (int j = 0; j < threads && why[0] == '\0'; ++j) {+ const int c = core_of(j);+ if (c < 0 && j < n_) std::snprintf(why, sizeof why, "worker %d has no core to be pinned to", j);+ else if (c < 0) std::snprintf(why, sizeof why, "the host thread has no core");+ else if ((node[(size_t) j] = cpu_numa_node(c)) < 0)+ std::snprintf(why, sizeof why, "CPU %d has no known NUMA node", c);+ }+ if (why[0] == '\0') {+ std::vector<int> nodes = node;+ std::sort(nodes.begin(), nodes.end());+ nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end());+ if (nodes.size() != 2) {+ std::snprintf(why, sizeof why, "the pool's threads span %zu NUMA node(s), the prototype needs exactly 2",+ nodes.size());+ } else {+ plan.nodes[0] = nodes[0];+ plan.nodes[1] = nodes[1];+ for (int j = 0; j < threads; ++j) {+ slice[(size_t) j] = node[(size_t) j] == nodes[1] ? 1 : 0;+ plan.cores[slice[(size_t) j]].push_back(core_of(j));+ }+ }+ }+#else+ std::snprintf(why, sizeof why, "the split needs Linux");+#endif+ }+ plan.why = why;+ plan.on = plan.why.empty();+ if (plan.on) {+ numa_n_ = 2;+ nsync_.reset(new NodeSync[2]);+ wslice_.assign(slice.begin(), slice.begin() + n_);+ hslice_ = host_works_ ? slice[(size_t) n_] : 0;+ slice_threads_.assign(2, 0);+ for (int k : slice) ++slice_threads_[(size_t) k];+ if (plan.test)+ std::fprintf(stderr, "strata cpu pool: NUMA split on (test, no real nodes): slice 0 = %d threads, "+ "slice 1 = %d threads\n", slice_threads_[0], slice_threads_[1]);+ else+ std::fprintf(stderr, "strata cpu pool: NUMA split on: slice 0 = node %d (%d threads), slice 1 = node %d "+ "(%d threads)\n", plan.nodes[0], slice_threads_[0], plan.nodes[1], slice_threads_[1]);+ } else {+ std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());+ }+ }+ g_numa_plan = std::move(plan);+ g_numa_owner = this;threads_.reserve((size_t) n_);for (int i = 0; i < n_; ++i) {const int core = pin ? (i < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) i] : -1) : -1;threads_.emplace_back([this, i, core] {strata::aux_cpus::note_owned_thread(); // --aux-cpus never moves a worker- pin_this_thread(core, i);+ // a failed pin only costs locality under the split: the worker keeps the slice it was given+ if (!pin_this_thread(core, i) && core >= 0 && numa_n_ > 0)+ std::fprintf(stderr, "strata cpu pool: NUMA split: worker %d not pinned to CPU %d, its slice's rows may "+ "be read remotely\n", i, core);worker(i);});}@@ -650,6 +746,10 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity afExpertPool::~ExpertPool() {const ExpertPool* self = this;g_diag_pool.compare_exchange_strong(self, nullptr);+ if (g_numa_owner == this) {+ g_numa_plan = NumaSplitPlan{};+ g_numa_owner = nullptr;+ }stop_.store(true, std::memory_order_release);// Bump the epoch so a PARKED worker notices the stop flag rather than sleeping through it.publish();@@ -675,7 +775,7 @@ void ExpertPool::worker(int id) {// on the RETURN from a drain leaves `parked_` at 0 until each worker has finished one batch, and the first// `run()` - which waits for `parked_ == n_` before publishing - then deadlocks. It deadlocks on the very// first call, which is the good case; a version that deadlocked on the second would be far worse.- parked_.fetch_add(1, std::memory_order_acq_rel);+ parked_of(id).fetch_add(1, std::memory_order_acq_rel);for (;;) {// Park: wait for work. `_mm_pause` rather than a bare spin because it yields the pipeline to the// sibling hyperthread; `epoch_` is bumped once per LAYER, not once per expert, so most of these@@ -724,7 +824,7 @@ void ExpertPool::worker(int id) {if (stop_.load(std::memory_order_acquire)) return;// acquire: the batch this epoch published (`head`, and the description before it) is visible from hereseen = epoch_.load(std::memory_order_acquire);- parked_.fetch_sub(1, std::memory_order_acq_rel); // leaving the park+ parked_of(id).fetch_sub(1, std::memory_order_acq_rel); // leaving the park// Drain: one claim per iteration, so a slow worker takes fewer experts and a fast one takes more.// Every job is the same size (all experts are 1,382,400 bytes), so there is nothing to schedule. Only@@ -732,7 +832,7 @@ void ExpertPool::worker(int id) {wstate_[(size_t) id].store(kBetween, std::memory_order_relaxed);drain(id, scratch_[(size_t) id], seen);wstate_[(size_t) id].store(kParked, std::memory_order_relaxed);- parked_.fetch_add(1, std::memory_order_acq_rel); // back at the park+ parked_of(id).fetch_add(1, std::memory_order_acq_rel); // back at the park}}@@ -747,6 +847,56 @@ int ExpertPool::claim(uint32_t epoch) {}}+int ExpertPool::claim_slice(uint32_t epoch, int slice) {+ NodeSync& ns = nsync_[(size_t) slice];+ uint64_t h = ns.head.load(std::memory_order_acquire);+ for (;;) {+ if ((uint32_t) (h >> 32) != epoch) return -1; // not this batch, or not a mode-5/6 batch+ const uint32_t n = (uint32_t) (h >> 16) & 0xffffu, i = (uint32_t) h & 0xffffu;+ if (i >= n) return -1; // the slice is exhausted+ if (ns.head.compare_exchange_weak(h, h + 1, std::memory_order_acq_rel, std::memory_order_acquire))+ return ns.base + (int) i; // the description is stable from here+ }+}++uint32_t ExpertPool::parked_count() const {+ if (numa_n_ == 0) return parked_.load(std::memory_order_acquire);+ uint32_t p = 0; // each slice's counter is bounded by its own workers, so the sum is n_ only when each is full+ for (int k = 0; k < numa_n_; ++k) p += nsync_[(size_t) k].parked.load(std::memory_order_acquire);+ return p;+}++std::string ExpertPool::numa_counters() const {+ std::string s;+ for (int k = 0; k < numa_n_; ++k) {+ const NodeSync& ns = nsync_[(size_t) k];+ const uint64_t h = ns.head.load();+ char buf[200];+ std::snprintf(buf, sizeof buf, "%s slice %d: batch epoch %u, %u of %u claimed, %u done, %u parked, %d threads",+ k == 0 ? " [" : ";", k, (uint32_t) (h >> 32), (uint32_t) h & 0xffffu, (uint32_t) (h >> 16) & 0xffffu,+ ns.done.load(), ns.parked.load(), slice_threads_[(size_t) k]);+ s += buf;+ }+ if (numa_n_ > 0) s += "]";+ return s;+}++int ExpertPool::numa_plan_phase(int per, int nb) {+ int base = 0;+ for (int k = 0; k < numa_n_; ++k) {+ NodeSync& ns = nsync_[(size_t) k];+ ns.row0 = per * k / numa_n_;+ ns.h = per * (k + 1) / numa_n_ - ns.row0;+ ns.rows = (int64_t) nb * ns.h;+ // about three tasks per thread of the slice's node; an explicit --pool-tasks is shared between the slices+ const int64_t want = tasks_ == 0 ? 3 * (int64_t) slice_threads_[(size_t) k] : (int64_t) (std::max)(1, tasks_ / 2);+ ns.tasks = (int) (std::min)(want, ns.rows);+ ns.base = base;+ base += ns.tasks;+ }+ return base;+}+uint32_t ExpertPool::begin_batch(int n) {if (n < 0 || n > 0xffff) {std::fprintf(stderr, "strata: expert pool batch of %d jobs is out of range\n", n);@@ -754,9 +904,21 @@ uint32_t ExpertPool::begin_batch(int n) {}// Every job of the previous batch has completed (`wait_done`), and a claim of it can no longer succeed, so// nothing adds to `done` until this batch's first claim - which the release below orders after the reset.- done_.store(0, std::memory_order_relaxed);const uint32_t e = epoch_.load(std::memory_order_relaxed) + 1; // only the host bumps the epoch- head_.store(pack_head(e, (uint32_t) n, 0), std::memory_order_release);+ if (numa_phase()) {+ // every slice's head, even a slice with no task, and never `head_`: for this epoch only the slice heads+ // carry `e`, so a claim on `head_` fails, and every head left from an earlier batch is exhausted (pool.hpp)+ for (int k = 0; k < numa_n_; ++k) {+ NodeSync& ns = nsync_[(size_t) k];+ ns.done.store(0, std::memory_order_relaxed);+ ns.drained_ns.store(0, std::memory_order_relaxed);+ ns.head.store(pack_head(e, (uint32_t) ns.tasks, 0), std::memory_order_release);+ }+ pub_ns_ = now_ns();+ } else {+ done_.store(0, std::memory_order_relaxed);+ head_.store(pack_head(e, (uint32_t) n, 0), std::memory_order_release);+ }publish();return e;}@@ -766,15 +928,15 @@ void ExpertPool::wait_parked(const char* what) {hstate_ms_.store(now_ms(), std::memory_order_relaxed);uint32_t spins = 0;std::chrono::steady_clock::time_point t0{};- while (parked_.load(std::memory_order_acquire) != (uint32_t) n_) {+ while (parked_count() != (uint32_t) n_) {_mm_pause();if ((++spins & 1023u) != 0) continue;const auto now = std::chrono::steady_clock::now();if (spins == 1024u) t0 = now;else if (now - t0 > kStall) {- std::fprintf(stderr, "strata: the CPU expert pool stalled %s (%u of %d workers parked) - stopping the engine "+ std::fprintf(stderr, "strata: the CPU expert pool stalled %s (%u of %d workers parked)%s - stopping the engine ""so the server can start it again (issue #29)\n",- what, parked_.load(), n_);+ what, parked_count(), n_, numa_counters().c_str());strata::core::release_gpu_waits(stderr); // #267: the GPU may be spinning on this layer's flagstd::fflush(stderr);std::abort();@@ -788,16 +950,20 @@ void ExpertPool::wait_done(int n) {uint32_t spins = 0, seen = 0;std::chrono::steady_clock::time_point t0{};for (;;) {- const uint32_t d = done_.load(std::memory_order_acquire);+ uint32_t d = 0;+ if (numa_phase()) // the slices' counts, each bounded by its tasks: the sum is n only when every slice is done+ for (int k = 0; k < numa_n_; ++k) d += nsync_[(size_t) k].done.load(std::memory_order_acquire);+ else+ d = done_.load(std::memory_order_acquire);if (d >= (uint32_t) n) return; // `>=`: never a wait that an overshoot outlives_mm_pause();if ((++spins & 1023u) != 0) continue;const auto now = std::chrono::steady_clock::now();if (spins == 1024u || d != seen) { t0 = now; seen = d; } // progress restarts the clockelse if (now - t0 > kStall) {- std::fprintf(stderr, "strata: the CPU expert pool stalled: %u of %d jobs done, %u of %d workers parked - "+ std::fprintf(stderr, "strata: the CPU expert pool stalled: %u of %d jobs done, %u of %d workers parked%s - ""stopping the engine so the server can start it again (issue #29)\n",- d, n, parked_.load(), n_);+ d, n, parked_count(), n_, numa_counters().c_str());strata::core::release_gpu_waits(stderr); // #267std::fflush(stderr);std::abort();@@ -806,9 +972,13 @@ void ExpertPool::wait_done(int n) {}void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {- (void) id;+ const int s = numa_n_ == 0 ? 0 : id >= 0 ? wslice_[(size_t) id] : hslice_;for (;;) {- const int ci = claim(epoch);+ // STRATA_NUMA_SPLIT: a mode-5/6 batch lives on the slice heads, any other on `head_`; the epoch tag makes the+ // other family's claim fail, so `mode_` is never read before a claim has succeeded (pool.hpp)+ int ci = numa_n_ > 0 ? claim_slice(epoch, s) : -1;+ const bool sliced = ci >= 0;+ if (!sliced) ci = claim(epoch);if (ci < 0) break;const uint32_t i = (uint32_t) ci;if (id >= 0) wstate_[(size_t) id].store(ci, std::memory_order_relaxed);@@ -827,11 +997,21 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {} else if (mode_ >= 5) {// plan v0.3 P6: native layers, 5 = gate/up rows, 6 = down rows (the layer's own n_ff / n_embd: Flash-Next's// FF / H, or another family's smaller expert)+ // Task j of T over R rows of `h` per expert from matrix row `row0`: the whole phase (R = mrows_, h = per,+ // row0 = 0) or, under the NUMA split, this thread's slice (slice-local row g -> expert g / h, matrix row+ // row0 + g % h)const int per = mode_ == 5 ? (int) nfmt_->n_ff : (int) nfmt_->n_embd;- const int64_t g0 = mrows_ * (int64_t) i / mtasks_, g1 = mrows_ * (int64_t) (i + 1) / mtasks_;- for (int64_t r = g0; r < g1;) {- const int e = (int) (r / per), r0 = (int) (r % per);- const int r1 = (int) std::min<int64_t>(per, r0 + (g1 - r));+ int64_t R = mrows_, T = mtasks_, j = i;+ int h = per, row0 = 0;+ if (sliced) {+ const NodeSync& ns = nsync_[(size_t) s];+ R = ns.rows; T = ns.tasks; h = ns.h; row0 = ns.row0; j = ci - ns.base;+ }+ const int64_t g0 = R * j / T, g1 = R * (j + 1) / T;+ for (int64_t g = g0; g < g1;) {+ const int e = (int) (g / h), l0 = (int) (g % h);+ const int l1 = (int) std::min<int64_t>(h, l0 + (g1 - g));+ const int r0 = row0 + l0, r1 = row0 + l1;SplitBufMulti& sb = split_multi_[(size_t) e];const NativeFmt* const nfmt = mjobs_[e].fmt != nullptr ? mjobs_[e].fmt : nfmt_; // 149-cold: same types and rowsif (mode_ == 5 && q2_native_kernels(nfmt->gu_type)) {@@ -861,7 +1041,7 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {for (int t = 0; t < mjobs_[e].nt; ++t) hq[t] = sb.hq[t];native_down_rows(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);}- r += r1 - r0;+ g += l1 - l0;}} else {// plan v0.3 P6: an equal range of the phase's rows across ALL its experts (a range may span two)@@ -883,7 +1063,15 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {r += r1 - r0;}}- done_.fetch_add(1, std::memory_order_release);+ if (sliced) {+ NodeSync& ns = nsync_[(size_t) s];+ const uint32_t t = (uint32_t) ns.tasks;+ // the slice's last task stamps its drain time; read by the host once this thread has parked again+ if (ns.done.fetch_add(1, std::memory_order_release) + 1 == t)+ ns.drained_ns.store(now_ns(), std::memory_order_relaxed);+ } else {+ done_.fetch_add(1, std::memory_order_release);+ }}}@@ -895,6 +1083,14 @@ void ExpertPool::run_phase(int mode, int n_tasks) {if (host_works_) drain(-1, host_scratch_, e);wait_done(n_tasks);wait_parked("after a phase");+ // Each slice's stamp is stored after its last `done` increment, so `wait_done` can return before it lands; the+ // thread that stored it has parked again since (it left the park to claim), and that re-park is what the wait+ // above acquired. So the stamps are read here, not in `wait_done`.+ if (numa_phase())+ for (int k = 0; k < numa_n_; ++k) {+ const int64_t t = nsync_[(size_t) k].drained_ns.load(std::memory_order_acquire);+ if (t != 0) slice_ms_[k] += (double) (t - pub_ns_) / 1e6;+ }hstate_.store(kIdle, std::memory_order_relaxed);hstate_ms_.store(now_ms(), std::memory_order_relaxed);}@@ -967,7 +1163,7 @@ void ExpertPool::run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobsmjobs_ = jobs + b0;nfmt_ = &f;mrows_ = (int64_t) nb * f.n_ff;- mtasks_ = phase_tasks(mrows_);+ mtasks_ = numa_n_ > 0 ? numa_plan_phase((int) f.n_ff, nb) : phase_tasks(mrows_);const auto a = std::chrono::steady_clock::now();run_phase(5, mtasks_);const auto b = std::chrono::steady_clock::now();@@ -977,7 +1173,7 @@ void ExpertPool::run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobselse native_quant_h(f, split_multi_[(size_t) e].ff[t], split_multi_[(size_t) e].hq[t]);const auto c = std::chrono::steady_clock::now();mrows_ = (int64_t) nb * f.n_embd;- mtasks_ = phase_tasks(mrows_);+ mtasks_ = numa_n_ > 0 ? numa_plan_phase((int) f.n_embd, nb) : phase_tasks(mrows_);run_phase(6, mtasks_);const auto d = std::chrono::steady_clock::now();ms_multi_gu += std::chrono::duration<double, std::milli>(b - a).count();diff --git a/src/kernels/cpu/pool_tasks_test.cpp b/src/kernels/cpu/pool_tasks_test.cppindex dbb5e754a..bf419cc3d 100644--- a/src/kernels/cpu/pool_tasks_test.cpp+++ b/src/kernels/cpu/pool_tasks_test.cpp@@ -14,6 +14,21 @@namespace cpu = strata::kernels::cpu;+static void put_env(const char* k, const char* v) {+#if defined(_WIN32)+ _putenv_s(k, v);+#else+ setenv(k, v, 1);+#endif+}+static void unset_env(const char* k) {+#if defined(_WIN32)+ _putenv_s(k, "");+#else+ unsetenv(k);+#endif+}+int main(int argc, char** argv) {if (argc != 1 && argc != 4) return 2;// its activation quantiser is the AVX one at least (act_quant_any): a CPU without AVX (Westmere) would stop on an illegal instruction@@ -51,7 +66,7 @@ int main(int argc, char** argv) {cpu::native_quant_act(f, x.data(), act[t].data());cpu::act_quant_any(x.data(), cpu::H, qact[t]);}- int checks = 0;+ int checks = 0, split_checks = 0;for (int n : {1, 7, cpu::ExpertPool::kMaxSplitMulti + 1}) {std::vector<cpu::ExpertJobMulti> jobs(n);std::vector<float> out((size_t) n * cpu::MAXT * cpu::H);@@ -86,7 +101,38 @@ int main(int argc, char** argv) {++checks;}}+ // STRATA_NUMA_SPLIT=test: two slices without a real second node (thread j -> slice j % 2), every worker+ // sleeping between batches (spin 0) so the late-wake path of issue #29 runs constantly+ put_env("STRATA_NUMA_SPLIT", "test");+ put_env("STRATA_POOL_SPIN_US", "0");+ for (bool host : {false, true}) for (int workers : {1, 2, 3, 5})+ for (int tasks : {0, 1, 41, cpu::ExpertPool::kMaxTasks}) {+ cpu::ExpertPool pool(workers, false, host, cpu::PoolAffinity::All, tasks);+ const int slices = workers + (host ? 1 : 0) >= 2 ? 2 : 0;+ if (pool.numa_slices() != slices || cpu::active_numa_split().on != (slices == 2) ||+ (slices == 2 && !cpu::active_numa_split().test)) {+ std::fprintf(stderr, "numa-split slices %d, expected %d (workers=%d host=%d)\n",+ pool.numa_slices(), slices, workers, host);+ return 1;+ }+ for (int repeat = 0; repeat < 2; ++repeat) {+ for (const auto& job : jobs) for (int t = 0; t < job.nt; ++t)+ std::fill(job.out[t], job.out[t] + cpu::H, std::numeric_limits<float>::quiet_NaN());+ pool.run_split_multi_native(f, jobs.data(), n);+ if (std::memcmp(ref.data(), out.data(), out.size() * sizeof(float)) != 0) {+ std::fprintf(stderr, "numa-split mismatch n=%d workers=%d host=%d tasks=%d repeat=%d\n",+ n, workers, host, tasks, repeat);+ return 1;+ }+ ++split_checks;+ }+ pool.diag(stderr);+ }+ unset_env("STRATA_NUMA_SPLIT");+ unset_env("STRATA_POOL_SPIN_US");+ if (cpu::active_numa_split().on) { std::fprintf(stderr, "numa-split plan outlived its pool\n"); return 1; }}- std::printf("pool tasks: %d bitwise comparisons passed\n", checks);+ std::printf("pool tasks: %d bitwise comparisons passed, %d of them in NUMA-split mode\n", checks + split_checks,+ split_checks);return 0;}diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 711bbc663..ea7cb608d 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -16304,10 +16304,13 @@ int main(int argc, char** argv) {(double) (drive.d.multi_entries - entries0) / (double) (rounds * g.n_layers));if (rounds > 0)std::printf("%-24s gate/up %.3f quantize %.3f down %.3f ms/round; %.1f GB/s over the rows phases; "- "CPU pool call %.3f ms/round\n", "pool multi", pool.ms_multi_gu / rounds,+ "CPU pool call %.3f ms/round", "pool multi", pool.ms_multi_gu / rounds,pool.ms_multi_q / rounds, pool.ms_multi_down / rounds,(double) pool.multi_bytes / 1e6 / std::max(1e-9, pool.ms_multi_gu + pool.ms_multi_down),(drive.cpu_ms - pool_ms0) / rounds);+ if (rounds > 0 && pool.numa_slices() == 2) // STRATA_NUMA_SPLIT: each slice's drain, publish to its last task+ std::printf("; numa slices: s0 %.3f ms s1 %.3f ms", pool.numa_slice_ms(0) / rounds, pool.numa_slice_ms(1) / rounds);+ if (rounds > 0) std::printf("\n");if (rounds > 0)std::printf("%-24s plan %.3f activation quantize %.3f jobs %.3f run %.3f ms/round\n", "dispatch",drive.d.ms_plan / rounds, drive.d.ms_actq / rounds, drive.d.ms_jobs / rounds,