Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

feat: per-NUMA-node task slices and counters in the CPU expert pool (opt-in)

Baptiste Rajaut709f8dc63d5f
diff --git a/include/strata/kernels/cpu/pool.hpp b/include/strata/kernels/cpu/pool.hpp
index 36e8c89cc..9dcb349bd 100644
--- a/include/strata/kernels/cpu/pool.hpp
+++ b/include/strata/kernels/cpu/pool.hpp
@@ -38,6 +38,7 @@
#include <condition_variable>
#include <cstdint>
#include <mutex>
+#include <string>
#include <thread>
#include <vector>
@@ -153,6 +154,20 @@ struct ThreadAffinity {
ThreadAffinity pin_current_thread(int core);
void restore_thread_affinity(const ThreadAffinity& previous);
+/// STRATA_NUMA_SPLIT's decision for a pool (see `ExpertPool`'s NodeSync). `1` splits on the real NUMA nodes of
+/// the pinned threads (Linux, exactly 2 nodes); `test` splits by thread index (thread j -> slice j % 2) with no real
+/// node, for the parity test.
+struct NumaSplitPlan {
+ bool on = false; ///< split enabled
+ bool test = false; ///< STRATA_NUMA_SPLIT=test: no real nodes, never places memory
+ int nodes[2] = {-1, -1}; ///< real node id of slice 0 and 1 (ascending); -1 in test mode
+ std::vector<int> cores[2]; ///< the cores the pool pinned for each slice (workers + host)
+ std::string why; ///< reason when off ("" when on, or when the env var is unset)
+};
+/// The live pool's decision (set by the ExpertPool constructor, reset by its destructor); `on == false` when no
+/// pool exists or the split is off.
+const NumaSplitPlan& active_numa_split();
+
class ExpertPool {
public:
/// `n_workers <= 0` means "every physical core except the first". Workers are pinned to physical cores
@@ -185,6 +200,10 @@ public:
const char* plan_limit() const { return plan_limit_; }
int plan_cores() const { return plan_cores_; }
int plan_logical() const { return plan_logical_; }
+ /// STRATA_NUMA_SPLIT: 2 when the native row phases are split per NUMA node, 0 when off.
+ int numa_slices() const { return numa_n_; }
+ /// Accumulated drain time of slice `s` over the mode-5/6 phases, ms: publish to the slice's last `done`.
+ double numa_slice_ms(int s) const { return s >= 0 && s < numa_n_ ? slice_ms_[s] : 0.0; }
/// Whether the host thread also drains. Reported at startup, because "the engine adapts to the machine it
/// is on" is only true if the engine says which adaptation it took.
bool host_works() const { return host_works_; }
@@ -326,6 +345,48 @@ private:
};
const NativeFmt* nfmt_ = nullptr;
std::vector<SplitBufMulti> split_multi_;
+ // STRATA_NUMA_SPLIT=1 (opt-in, Linux, exactly 2 nodes): in the native row phases (modes 5/6) every expert's rows
+ // are cut into one slice per NUMA node (slice k = rows [per*k/2, per*(k+1)/2) of the gate/up or down matrix, the
+ // part the resident complement places on node k in step B), and each node's threads claim and count on their own
+ // slice's counters; in every mode they park on their own `parked`. Off (`numa_n_ == 0`, the decision is final in
+ // the constructor): none of this is touched and the pool is as before.
+ //
+ // THE CLAIM PROTOCOL IS issue #29's, ONCE PER SLICE. Each slice head packs `epoch | njobs | index` and is claimed
+ // with the same CAS as `head_`. A mode-5/6 batch stores every slice head with its epoch and leaves `head_` alone;
+ // any other batch stores `head_` and leaves the slice heads alone. So for batch `e` exactly one family of heads
+ // carries `e`, and a stale head is always exhausted (`wait_done` saw every task of it done). A worker therefore
+ // never reads `mode_` to pick a counter: it tries its slice head, then `head_`, and the epoch tag makes the wrong
+ // one fail - as it makes a late worker's claim fail.
+ struct NodeSync {
+ alignas(64) std::atomic<uint64_t> head{0}; // as `head_`, for this slice of the phase
+ alignas(64) std::atomic<uint32_t> done{0}; // as `done_`
+ alignas(64) std::atomic<uint32_t> parked{0}; // as `parked_`, this slice's workers only
+ alignas(64) std::atomic<int64_t> drained_ns{0}; // steady_clock ns when `done` reached `tasks` (0: not yet)
+ // the batch's description, written by the host before the head's release store, read after a claim
+ alignas(64) int base = 0; // the slice's first task index in the phase (task ids are global for `diag`)
+ int tasks = 0; // T: the slice's task count
+ int h = 0; // rows of each expert in the slice
+ int row0 = 0; // the slice's first matrix row (per*k/2)
+ int64_t rows = 0; // R = experts * h
+ };
+ // CBA: the NodeSync blocks live wherever `new` puts them (the host's node); per-node pages if the remote home
+ // of a line ever shows up in the counters.
+ std::unique_ptr<NodeSync[]> nsync_;
+ int numa_n_ = 0; // slices (2 when on), 0 = off
+ std::vector<int> wslice_; // each worker's slice
+ int hslice_ = 0; // the host's slice (when it drains)
+ std::vector<int> slice_threads_; // threads draining each slice (the host counted in its own)
+ int64_t pub_ns_ = 0; // the current mode-5/6 phase's publish time (host only)
+ double slice_ms_[2] = {0.0, 0.0}; // accumulated per-slice drain time (host only)
+ bool numa_phase() const { return numa_n_ > 0 && mode_ >= 5; }
+ /// The claim of `claim()` on slice `slice`'s head: `base + i`, or -1.
+ int claim_slice(uint32_t epoch, int slice);
+ std::atomic<uint32_t>& parked_of(int id) { return numa_n_ > 0 ? nsync_[(size_t) wslice_[(size_t) id]].parked : parked_; }
+ uint32_t parked_count() const;
+ /// Fill the slices' descriptions for a mode-5/6 phase of `nb` experts of `per` rows; returns the total tasks.
+ int numa_plan_phase(int per, int nb);
+ /// Per-slice counters for the stall messages and `diag` ("" when off).
+ std::string numa_counters() const;
PoolAffinity affinity_ = PoolAffinity::All;
CpuTopology topo_;
};
diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cpp
index 80161b9a2..772a68781 100644
--- a/src/kernels/cpu/pool.cpp
+++ b/src/kernels/cpu/pool.cpp
@@ -27,6 +27,7 @@
#else
#include <pthread.h>
#include <sched.h>
+#include <unistd.h>
#include "pool_affinity_linux.hpp"
#endif
@@ -530,6 +531,19 @@ bool pin_this_thread(int core, [[maybe_unused]] int worker = -1) {
#endif
}
+#if defined(__linux__)
+// The NUMA node of a logical CPU (sysfs), or -1.
+int cpu_numa_node(int cpu) {
+ if (cpu < 0) return -1;
+ char path[96];
+ for (int node = 0; node < 64; ++node) {
+ std::snprintf(path, sizeof path, "/sys/devices/system/cpu/cpu%d/node%d", cpu, node);
+ if (access(path, F_OK) == 0) return node;
+ }
+ return -1;
+}
+#endif
+
} // namespace
ThreadAffinity pin_current_thread(int core) {
@@ -583,13 +597,24 @@ int64_t now_ms() {
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch())
.count();
}
+int64_t now_ns() {
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch())
+ .count();
+}
+// CBA: one plain global, written by the constructor and destructor of the pool that owns it; fine while pools are
+// built and destroyed on one thread (generate's measure pools, then the main one), an atomic owner if that changes.
+NumaSplitPlan g_numa_plan;
+const ExpertPool* g_numa_owner = nullptr;
} // namespace
+const NumaSplitPlan& active_numa_split() { return g_numa_plan; }
+
void ExpertPool::diag(std::FILE* f) const {
const uint64_t h = head_.load();
std::fprintf(f, " expert pool: epoch %u, batch epoch %u: %u of %u jobs claimed, %u done; %u of %d workers parked, "
"%u sleeping; mode %d\n", epoch_.load(), (uint32_t) (h >> 32), (uint32_t) h & 0xffffu,
- (uint32_t) (h >> 16) & 0xffffu, done_.load(), parked_.load(), n_, sleepers_.load(), mode_);
+ (uint32_t) (h >> 16) & 0xffffu, done_.load(), parked_count(), n_, sleepers_.load(), mode_);
+ if (numa_n_ > 0) std::fprintf(f, " expert pool NUMA slices:%s\n", numa_counters().c_str());
std::fprintf(f, " expert pool threads:");
for (int i = 0; i < n_; ++i) {
const int32_t s = wstate_[(size_t) i].load();
@@ -612,20 +637,20 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
if (const char* e = std::getenv("STRATA_POOL_SPIN_US")) // a test knob; see kSpinBeforeSleep
spin_before_sleep_ = std::chrono::microseconds((std::max)(0, std::atoi(e)));
const PoolPlanInput plan_in = pool_plan_input(topo_);
- const PoolPlan plan = plan_pool_workers(plan_in);
+ const PoolPlan wplan = plan_pool_workers(plan_in);
if (n_workers > 0) {
n_ = n_workers;
// recommend, never force: a count above what this CPU uses well runs as configured, with a plain warning
static std::atomic<bool> warned{false};
- const std::string why = pool_workers_warning(n_workers, plan_in, plan);
+ const std::string why = pool_workers_warning(n_workers, plan_in, wplan);
if (!why.empty() && !warned.exchange(true)) std::fprintf(stderr, "strata cpu pool: warning: %s\n", why.c_str());
} else if (topo_.is_hybrid && affinity_ != PoolAffinity::All) {
n_ = (std::max)(1, topo_.p_cores - 1);
} else {
- n_ = (std::min)(plan.workers, (std::max)(1, (int) topo_.worker_cores.size()));
- plan_limit_ = plan.limit;
+ n_ = (std::min)(wplan.workers, (std::max)(1, (int) topo_.worker_cores.size()));
+ plan_limit_ = wplan.limit;
}
- plan_cores_ = plan.cores;
+ plan_cores_ = wplan.cores;
plan_logical_ = plan_in.logical;
if (n_ < 1) n_ = 1;
scratch_.resize((size_t) n_);
@@ -636,12 +661,83 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
strata::core::diag_pool_fn().store(&diag_active_pool);
split_.resize((size_t) kMaxSplit);
split_multi_.resize((size_t) kMaxSplitMulti);
+ // STRATA_NUMA_SPLIT: decided here, before any worker starts (a worker counts into its slice's `parked` from its
+ // first instruction), and never revised. Unset: the plan stays empty and nothing below runs.
+ NumaSplitPlan plan;
+ if (const char* env = std::getenv("STRATA_NUMA_SPLIT"); env != nullptr && *env != '\0' && std::strcmp(env, "0") != 0) {
+ plan.test = std::strcmp(env, "test") == 0;
+ const int threads = n_ + (host_works_ ? 1 : 0); // thread j < n_: worker j; j == n_: the host
+ [[maybe_unused]] auto core_of = [&](int j) {
+ if (j == n_) return topo_.host_core;
+ return pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1;
+ };
+ std::vector<int> slice((size_t) threads, 0);
+ char why[160] = "";
+ if (threads < 2) {
+ std::snprintf(why, sizeof why, "the pool has %d thread, the split needs 2 or more", threads);
+ } else if (plan.test) {
+ for (int j = 0; j < threads; ++j) slice[(size_t) j] = j % 2;
+ } else {
+#if defined(__linux__)
+ std::vector<int> node((size_t) threads, -1);
+ if (!pin) std::snprintf(why, sizeof why, "the pool's threads are not pinned");
+ for (int j = 0; j < threads && why[0] == '\0'; ++j) {
+ const int c = core_of(j);
+ if (c < 0 && j < n_) std::snprintf(why, sizeof why, "worker %d has no core to be pinned to", j);
+ else if (c < 0) std::snprintf(why, sizeof why, "the host thread has no core");
+ else if ((node[(size_t) j] = cpu_numa_node(c)) < 0)
+ std::snprintf(why, sizeof why, "CPU %d has no known NUMA node", c);
+ }
+ if (why[0] == '\0') {
+ std::vector<int> nodes = node;
+ std::sort(nodes.begin(), nodes.end());
+ nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end());
+ if (nodes.size() != 2) {
+ std::snprintf(why, sizeof why, "the pool's threads span %zu NUMA node(s), the prototype needs exactly 2",
+ nodes.size());
+ } else {
+ plan.nodes[0] = nodes[0];
+ plan.nodes[1] = nodes[1];
+ for (int j = 0; j < threads; ++j) {
+ slice[(size_t) j] = node[(size_t) j] == nodes[1] ? 1 : 0;
+ plan.cores[slice[(size_t) j]].push_back(core_of(j));
+ }
+ }
+ }
+#else
+ std::snprintf(why, sizeof why, "the split needs Linux");
+#endif
+ }
+ plan.why = why;
+ plan.on = plan.why.empty();
+ if (plan.on) {
+ numa_n_ = 2;
+ nsync_.reset(new NodeSync[2]);
+ wslice_.assign(slice.begin(), slice.begin() + n_);
+ hslice_ = host_works_ ? slice[(size_t) n_] : 0;
+ slice_threads_.assign(2, 0);
+ for (int k : slice) ++slice_threads_[(size_t) k];
+ if (plan.test)
+ std::fprintf(stderr, "strata cpu pool: NUMA split on (test, no real nodes): slice 0 = %d threads, "
+ "slice 1 = %d threads\n", slice_threads_[0], slice_threads_[1]);
+ else
+ std::fprintf(stderr, "strata cpu pool: NUMA split on: slice 0 = node %d (%d threads), slice 1 = node %d "
+ "(%d threads)\n", plan.nodes[0], slice_threads_[0], plan.nodes[1], slice_threads_[1]);
+ } else {
+ std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());
+ }
+ }
+ g_numa_plan = std::move(plan);
+ g_numa_owner = this;
threads_.reserve((size_t) n_);
for (int i = 0; i < n_; ++i) {
const int core = pin ? (i < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) i] : -1) : -1;
threads_.emplace_back([this, i, core] {
strata::aux_cpus::note_owned_thread(); // --aux-cpus never moves a worker
- pin_this_thread(core, i);
+ // a failed pin only costs locality under the split: the worker keeps the slice it was given
+ if (!pin_this_thread(core, i) && core >= 0 && numa_n_ > 0)
+ std::fprintf(stderr, "strata cpu pool: NUMA split: worker %d not pinned to CPU %d, its slice's rows may "
+ "be read remotely\n", i, core);
worker(i);
});
}
@@ -650,6 +746,10 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
ExpertPool::~ExpertPool() {
const ExpertPool* self = this;
g_diag_pool.compare_exchange_strong(self, nullptr);
+ if (g_numa_owner == this) {
+ g_numa_plan = NumaSplitPlan{};
+ g_numa_owner = nullptr;
+ }
stop_.store(true, std::memory_order_release);
// Bump the epoch so a PARKED worker notices the stop flag rather than sleeping through it.
publish();
@@ -675,7 +775,7 @@ void ExpertPool::worker(int id) {
// on the RETURN from a drain leaves `parked_` at 0 until each worker has finished one batch, and the first
// `run()` - which waits for `parked_ == n_` before publishing - then deadlocks. It deadlocks on the very
// first call, which is the good case; a version that deadlocked on the second would be far worse.
- parked_.fetch_add(1, std::memory_order_acq_rel);
+ parked_of(id).fetch_add(1, std::memory_order_acq_rel);
for (;;) {
// Park: wait for work. `_mm_pause` rather than a bare spin because it yields the pipeline to the
// sibling hyperthread; `epoch_` is bumped once per LAYER, not once per expert, so most of these
@@ -724,7 +824,7 @@ void ExpertPool::worker(int id) {
if (stop_.load(std::memory_order_acquire)) return;
// acquire: the batch this epoch published (`head`, and the description before it) is visible from here
seen = epoch_.load(std::memory_order_acquire);
- parked_.fetch_sub(1, std::memory_order_acq_rel); // leaving the park
+ parked_of(id).fetch_sub(1, std::memory_order_acq_rel); // leaving the park
// Drain: one claim per iteration, so a slow worker takes fewer experts and a fast one takes more.
// Every job is the same size (all experts are 1,382,400 bytes), so there is nothing to schedule. Only
@@ -732,7 +832,7 @@ void ExpertPool::worker(int id) {
wstate_[(size_t) id].store(kBetween, std::memory_order_relaxed);
drain(id, scratch_[(size_t) id], seen);
wstate_[(size_t) id].store(kParked, std::memory_order_relaxed);
- parked_.fetch_add(1, std::memory_order_acq_rel); // back at the park
+ parked_of(id).fetch_add(1, std::memory_order_acq_rel); // back at the park
}
}
@@ -747,6 +847,56 @@ int ExpertPool::claim(uint32_t epoch) {
}
}
+int ExpertPool::claim_slice(uint32_t epoch, int slice) {
+ NodeSync& ns = nsync_[(size_t) slice];
+ uint64_t h = ns.head.load(std::memory_order_acquire);
+ for (;;) {
+ if ((uint32_t) (h >> 32) != epoch) return -1; // not this batch, or not a mode-5/6 batch
+ const uint32_t n = (uint32_t) (h >> 16) & 0xffffu, i = (uint32_t) h & 0xffffu;
+ if (i >= n) return -1; // the slice is exhausted
+ if (ns.head.compare_exchange_weak(h, h + 1, std::memory_order_acq_rel, std::memory_order_acquire))
+ return ns.base + (int) i; // the description is stable from here
+ }
+}
+
+uint32_t ExpertPool::parked_count() const {
+ if (numa_n_ == 0) return parked_.load(std::memory_order_acquire);
+ uint32_t p = 0; // each slice's counter is bounded by its own workers, so the sum is n_ only when each is full
+ for (int k = 0; k < numa_n_; ++k) p += nsync_[(size_t) k].parked.load(std::memory_order_acquire);
+ return p;
+}
+
+std::string ExpertPool::numa_counters() const {
+ std::string s;
+ for (int k = 0; k < numa_n_; ++k) {
+ const NodeSync& ns = nsync_[(size_t) k];
+ const uint64_t h = ns.head.load();
+ char buf[200];
+ std::snprintf(buf, sizeof buf, "%s slice %d: batch epoch %u, %u of %u claimed, %u done, %u parked, %d threads",
+ k == 0 ? " [" : ";", k, (uint32_t) (h >> 32), (uint32_t) h & 0xffffu, (uint32_t) (h >> 16) & 0xffffu,
+ ns.done.load(), ns.parked.load(), slice_threads_[(size_t) k]);
+ s += buf;
+ }
+ if (numa_n_ > 0) s += "]";
+ return s;
+}
+
+int ExpertPool::numa_plan_phase(int per, int nb) {
+ int base = 0;
+ for (int k = 0; k < numa_n_; ++k) {
+ NodeSync& ns = nsync_[(size_t) k];
+ ns.row0 = per * k / numa_n_;
+ ns.h = per * (k + 1) / numa_n_ - ns.row0;
+ ns.rows = (int64_t) nb * ns.h;
+ // about three tasks per thread of the slice's node; an explicit --pool-tasks is shared between the slices
+ const int64_t want = tasks_ == 0 ? 3 * (int64_t) slice_threads_[(size_t) k] : (int64_t) (std::max)(1, tasks_ / 2);
+ ns.tasks = (int) (std::min)(want, ns.rows);
+ ns.base = base;
+ base += ns.tasks;
+ }
+ return base;
+}
+
uint32_t ExpertPool::begin_batch(int n) {
if (n < 0 || n > 0xffff) {
std::fprintf(stderr, "strata: expert pool batch of %d jobs is out of range\n", n);
@@ -754,9 +904,21 @@ uint32_t ExpertPool::begin_batch(int n) {
}
// Every job of the previous batch has completed (`wait_done`), and a claim of it can no longer succeed, so
// nothing adds to `done` until this batch's first claim - which the release below orders after the reset.
- done_.store(0, std::memory_order_relaxed);
const uint32_t e = epoch_.load(std::memory_order_relaxed) + 1; // only the host bumps the epoch
- head_.store(pack_head(e, (uint32_t) n, 0), std::memory_order_release);
+ if (numa_phase()) {
+ // every slice's head, even a slice with no task, and never `head_`: for this epoch only the slice heads
+ // carry `e`, so a claim on `head_` fails, and every head left from an earlier batch is exhausted (pool.hpp)
+ for (int k = 0; k < numa_n_; ++k) {
+ NodeSync& ns = nsync_[(size_t) k];
+ ns.done.store(0, std::memory_order_relaxed);
+ ns.drained_ns.store(0, std::memory_order_relaxed);
+ ns.head.store(pack_head(e, (uint32_t) ns.tasks, 0), std::memory_order_release);
+ }
+ pub_ns_ = now_ns();
+ } else {
+ done_.store(0, std::memory_order_relaxed);
+ head_.store(pack_head(e, (uint32_t) n, 0), std::memory_order_release);
+ }
publish();
return e;
}
@@ -766,15 +928,15 @@ void ExpertPool::wait_parked(const char* what) {
hstate_ms_.store(now_ms(), std::memory_order_relaxed);
uint32_t spins = 0;
std::chrono::steady_clock::time_point t0{};
- while (parked_.load(std::memory_order_acquire) != (uint32_t) n_) {
+ while (parked_count() != (uint32_t) n_) {
_mm_pause();
if ((++spins & 1023u) != 0) continue;
const auto now = std::chrono::steady_clock::now();
if (spins == 1024u) t0 = now;
else if (now - t0 > kStall) {
- std::fprintf(stderr, "strata: the CPU expert pool stalled %s (%u of %d workers parked) - stopping the engine "
+ std::fprintf(stderr, "strata: the CPU expert pool stalled %s (%u of %d workers parked)%s - stopping the engine "
"so the server can start it again (issue #29)\n",
- what, parked_.load(), n_);
+ what, parked_count(), n_, numa_counters().c_str());
strata::core::release_gpu_waits(stderr); // #267: the GPU may be spinning on this layer's flag
std::fflush(stderr);
std::abort();
@@ -788,16 +950,20 @@ void ExpertPool::wait_done(int n) {
uint32_t spins = 0, seen = 0;
std::chrono::steady_clock::time_point t0{};
for (;;) {
- const uint32_t d = done_.load(std::memory_order_acquire);
+ uint32_t d = 0;
+ if (numa_phase()) // the slices' counts, each bounded by its tasks: the sum is n only when every slice is done
+ for (int k = 0; k < numa_n_; ++k) d += nsync_[(size_t) k].done.load(std::memory_order_acquire);
+ else
+ d = done_.load(std::memory_order_acquire);
if (d >= (uint32_t) n) return; // `>=`: never a wait that an overshoot outlives
_mm_pause();
if ((++spins & 1023u) != 0) continue;
const auto now = std::chrono::steady_clock::now();
if (spins == 1024u || d != seen) { t0 = now; seen = d; } // progress restarts the clock
else if (now - t0 > kStall) {
- std::fprintf(stderr, "strata: the CPU expert pool stalled: %u of %d jobs done, %u of %d workers parked - "
+ std::fprintf(stderr, "strata: the CPU expert pool stalled: %u of %d jobs done, %u of %d workers parked%s - "
"stopping the engine so the server can start it again (issue #29)\n",
- d, n, parked_.load(), n_);
+ d, n, parked_count(), n_, numa_counters().c_str());
strata::core::release_gpu_waits(stderr); // #267
std::fflush(stderr);
std::abort();
@@ -806,9 +972,13 @@ void ExpertPool::wait_done(int n) {
}
void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {
- (void) id;
+ const int s = numa_n_ == 0 ? 0 : id >= 0 ? wslice_[(size_t) id] : hslice_;
for (;;) {
- const int ci = claim(epoch);
+ // STRATA_NUMA_SPLIT: a mode-5/6 batch lives on the slice heads, any other on `head_`; the epoch tag makes the
+ // other family's claim fail, so `mode_` is never read before a claim has succeeded (pool.hpp)
+ int ci = numa_n_ > 0 ? claim_slice(epoch, s) : -1;
+ const bool sliced = ci >= 0;
+ if (!sliced) ci = claim(epoch);
if (ci < 0) break;
const uint32_t i = (uint32_t) ci;
if (id >= 0) wstate_[(size_t) id].store(ci, std::memory_order_relaxed);
@@ -827,11 +997,21 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {
} else if (mode_ >= 5) {
// plan v0.3 P6: native layers, 5 = gate/up rows, 6 = down rows (the layer's own n_ff / n_embd: Flash-Next's
// FF / H, or another family's smaller expert)
+ // Task j of T over R rows of `h` per expert from matrix row `row0`: the whole phase (R = mrows_, h = per,
+ // row0 = 0) or, under the NUMA split, this thread's slice (slice-local row g -> expert g / h, matrix row
+ // row0 + g % h)
const int per = mode_ == 5 ? (int) nfmt_->n_ff : (int) nfmt_->n_embd;
- const int64_t g0 = mrows_ * (int64_t) i / mtasks_, g1 = mrows_ * (int64_t) (i + 1) / mtasks_;
- for (int64_t r = g0; r < g1;) {
- const int e = (int) (r / per), r0 = (int) (r % per);
- const int r1 = (int) std::min<int64_t>(per, r0 + (g1 - r));
+ int64_t R = mrows_, T = mtasks_, j = i;
+ int h = per, row0 = 0;
+ if (sliced) {
+ const NodeSync& ns = nsync_[(size_t) s];
+ R = ns.rows; T = ns.tasks; h = ns.h; row0 = ns.row0; j = ci - ns.base;
+ }
+ const int64_t g0 = R * j / T, g1 = R * (j + 1) / T;
+ for (int64_t g = g0; g < g1;) {
+ const int e = (int) (g / h), l0 = (int) (g % h);
+ const int l1 = (int) std::min<int64_t>(h, l0 + (g1 - g));
+ const int r0 = row0 + l0, r1 = row0 + l1;
SplitBufMulti& sb = split_multi_[(size_t) e];
const NativeFmt* const nfmt = mjobs_[e].fmt != nullptr ? mjobs_[e].fmt : nfmt_; // 149-cold: same types and rows
if (mode_ == 5 && q2_native_kernels(nfmt->gu_type)) {
@@ -861,7 +1041,7 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {
for (int t = 0; t < mjobs_[e].nt; ++t) hq[t] = sb.hq[t];
native_down_rows(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);
}
- r += r1 - r0;
+ g += l1 - l0;
}
} else {
// plan v0.3 P6: an equal range of the phase's rows across ALL its experts (a range may span two)
@@ -883,7 +1063,15 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {
r += r1 - r0;
}
}
- done_.fetch_add(1, std::memory_order_release);
+ if (sliced) {
+ NodeSync& ns = nsync_[(size_t) s];
+ const uint32_t t = (uint32_t) ns.tasks;
+ // the slice's last task stamps its drain time; read by the host once this thread has parked again
+ if (ns.done.fetch_add(1, std::memory_order_release) + 1 == t)
+ ns.drained_ns.store(now_ns(), std::memory_order_relaxed);
+ } else {
+ done_.fetch_add(1, std::memory_order_release);
+ }
}
}
@@ -895,6 +1083,14 @@ void ExpertPool::run_phase(int mode, int n_tasks) {
if (host_works_) drain(-1, host_scratch_, e);
wait_done(n_tasks);
wait_parked("after a phase");
+ // Each slice's stamp is stored after its last `done` increment, so `wait_done` can return before it lands; the
+ // thread that stored it has parked again since (it left the park to claim), and that re-park is what the wait
+ // above acquired. So the stamps are read here, not in `wait_done`.
+ if (numa_phase())
+ for (int k = 0; k < numa_n_; ++k) {
+ const int64_t t = nsync_[(size_t) k].drained_ns.load(std::memory_order_acquire);
+ if (t != 0) slice_ms_[k] += (double) (t - pub_ns_) / 1e6;
+ }
hstate_.store(kIdle, std::memory_order_relaxed);
hstate_ms_.store(now_ms(), std::memory_order_relaxed);
}
@@ -967,7 +1163,7 @@ void ExpertPool::run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobs
mjobs_ = jobs + b0;
nfmt_ = &f;
mrows_ = (int64_t) nb * f.n_ff;
- mtasks_ = phase_tasks(mrows_);
+ mtasks_ = numa_n_ > 0 ? numa_plan_phase((int) f.n_ff, nb) : phase_tasks(mrows_);
const auto a = std::chrono::steady_clock::now();
run_phase(5, mtasks_);
const auto b = std::chrono::steady_clock::now();
@@ -977,7 +1173,7 @@ void ExpertPool::run_split_multi_native(const NativeFmt& f, ExpertJobMulti* jobs
else native_quant_h(f, split_multi_[(size_t) e].ff[t], split_multi_[(size_t) e].hq[t]);
const auto c = std::chrono::steady_clock::now();
mrows_ = (int64_t) nb * f.n_embd;
- mtasks_ = phase_tasks(mrows_);
+ mtasks_ = numa_n_ > 0 ? numa_plan_phase((int) f.n_embd, nb) : phase_tasks(mrows_);
run_phase(6, mtasks_);
const auto d = std::chrono::steady_clock::now();
ms_multi_gu += std::chrono::duration<double, std::milli>(b - a).count();
diff --git a/src/kernels/cpu/pool_tasks_test.cpp b/src/kernels/cpu/pool_tasks_test.cpp
index dbb5e754a..bf419cc3d 100644
--- a/src/kernels/cpu/pool_tasks_test.cpp
+++ b/src/kernels/cpu/pool_tasks_test.cpp
@@ -14,6 +14,21 @@
namespace cpu = strata::kernels::cpu;
+static void put_env(const char* k, const char* v) {
+#if defined(_WIN32)
+ _putenv_s(k, v);
+#else
+ setenv(k, v, 1);
+#endif
+}
+static void unset_env(const char* k) {
+#if defined(_WIN32)
+ _putenv_s(k, "");
+#else
+ unsetenv(k);
+#endif
+}
+
int main(int argc, char** argv) {
if (argc != 1 && argc != 4) return 2;
// its activation quantiser is the AVX one at least (act_quant_any): a CPU without AVX (Westmere) would stop on an illegal instruction
@@ -51,7 +66,7 @@ int main(int argc, char** argv) {
cpu::native_quant_act(f, x.data(), act[t].data());
cpu::act_quant_any(x.data(), cpu::H, qact[t]);
}
- int checks = 0;
+ int checks = 0, split_checks = 0;
for (int n : {1, 7, cpu::ExpertPool::kMaxSplitMulti + 1}) {
std::vector<cpu::ExpertJobMulti> jobs(n);
std::vector<float> out((size_t) n * cpu::MAXT * cpu::H);
@@ -86,7 +101,38 @@ int main(int argc, char** argv) {
++checks;
}
}
+ // STRATA_NUMA_SPLIT=test: two slices without a real second node (thread j -> slice j % 2), every worker
+ // sleeping between batches (spin 0) so the late-wake path of issue #29 runs constantly
+ put_env("STRATA_NUMA_SPLIT", "test");
+ put_env("STRATA_POOL_SPIN_US", "0");
+ for (bool host : {false, true}) for (int workers : {1, 2, 3, 5})
+ for (int tasks : {0, 1, 41, cpu::ExpertPool::kMaxTasks}) {
+ cpu::ExpertPool pool(workers, false, host, cpu::PoolAffinity::All, tasks);
+ const int slices = workers + (host ? 1 : 0) >= 2 ? 2 : 0;
+ if (pool.numa_slices() != slices || cpu::active_numa_split().on != (slices == 2) ||
+ (slices == 2 && !cpu::active_numa_split().test)) {
+ std::fprintf(stderr, "numa-split slices %d, expected %d (workers=%d host=%d)\n",
+ pool.numa_slices(), slices, workers, host);
+ return 1;
+ }
+ for (int repeat = 0; repeat < 2; ++repeat) {
+ for (const auto& job : jobs) for (int t = 0; t < job.nt; ++t)
+ std::fill(job.out[t], job.out[t] + cpu::H, std::numeric_limits<float>::quiet_NaN());
+ pool.run_split_multi_native(f, jobs.data(), n);
+ if (std::memcmp(ref.data(), out.data(), out.size() * sizeof(float)) != 0) {
+ std::fprintf(stderr, "numa-split mismatch n=%d workers=%d host=%d tasks=%d repeat=%d\n",
+ n, workers, host, tasks, repeat);
+ return 1;
+ }
+ ++split_checks;
+ }
+ pool.diag(stderr);
+ }
+ unset_env("STRATA_NUMA_SPLIT");
+ unset_env("STRATA_POOL_SPIN_US");
+ if (cpu::active_numa_split().on) { std::fprintf(stderr, "numa-split plan outlived its pool\n"); return 1; }
}
- std::printf("pool tasks: %d bitwise comparisons passed\n", checks);
+ std::printf("pool tasks: %d bitwise comparisons passed, %d of them in NUMA-split mode\n", checks + split_checks,
+ split_checks);
return 0;
}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 711bbc663..ea7cb608d 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -16304,10 +16304,13 @@ int main(int argc, char** argv) {
(double) (drive.d.multi_entries - entries0) / (double) (rounds * g.n_layers));
if (rounds > 0)
std::printf("%-24s gate/up %.3f quantize %.3f down %.3f ms/round; %.1f GB/s over the rows phases; "
- "CPU pool call %.3f ms/round\n", "pool multi", pool.ms_multi_gu / rounds,
+ "CPU pool call %.3f ms/round", "pool multi", pool.ms_multi_gu / rounds,
pool.ms_multi_q / rounds, pool.ms_multi_down / rounds,
(double) pool.multi_bytes / 1e6 / std::max(1e-9, pool.ms_multi_gu + pool.ms_multi_down),
(drive.cpu_ms - pool_ms0) / rounds);
+ if (rounds > 0 && pool.numa_slices() == 2) // STRATA_NUMA_SPLIT: each slice's drain, publish to its last task
+ std::printf("; numa slices: s0 %.3f ms s1 %.3f ms", pool.numa_slice_ms(0) / rounds, pool.numa_slice_ms(1) / rounds);
+ if (rounds > 0) std::printf("\n");
if (rounds > 0)
std::printf("%-24s plan %.3f activation quantize %.3f jobs %.3f run %.3f ms/round\n", "dispatch",
drive.d.ms_plan / rounds, drive.d.ms_actq / rounds, drive.d.ms_jobs / rounds,