Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
CPU expert pool: default worker count from physical cores, host socket, quota and a cap; warning for a configured count above it; setup caps what it writes
Niko12215ece068f7e45
diff --git a/CMakeLists.txt b/CMakeLists.txtindex e178f26e0..5d2134a63 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -2086,6 +2086,10 @@ if(STRATA_BUILD_TESTS)add_executable(pool_test src/kernels/cpu/pool_test.cpp)target_link_libraries(pool_test PRIVATE strata_kernels_cpu)add_test(NAME pool_test COMMAND pool_test --selftest WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})+ # the default CPU expert worker count on mock CPUs (8c/16t, 16c/32t, 64c/128t, 2-socket 128c/256t, ...)+ add_executable(pool_plan_test src/kernels/cpu/pool_plan_test.cpp)+ target_include_directories(pool_plan_test PRIVATE include)+ add_test(NAME pool_plan_test COMMAND pool_plan_test)# issue #29: tiny batches with sleeping workers, as on a big-VRAM card; a hang fails itadd_executable(pool_stress src/kernels/cpu/pool_stress.cpp)target_link_libraries(pool_stress PRIVATE strata_kernels_cpu)diff --git a/include/strata/kernels/cpu/pool.hpp b/include/strata/kernels/cpu/pool.hppindex bf0a9bc04..36e8c89cc 100644--- a/include/strata/kernels/cpu/pool.hpp+++ b/include/strata/kernels/cpu/pool.hpp@@ -29,6 +29,7 @@#include "strata/kernels/cpu/expert.hpp"#include "strata/kernels/cpu/native_expert.hpp"+#include "strata/kernels/cpu/pool_plan.hpp"#include <atomic>#include <cstdio>@@ -82,8 +83,22 @@ struct CpuTopology {std::vector<int> worker_cores; ///< Ordered CPU IDs; on Windows, group * 64 + processor within the groupint host_core = -1; ///< Logical core reserved for host threadint host_sibling = -1; ///< The host core's SMT sibling (-1: none), the logical processor `Sibling` takes+ /// For the default worker count (pool_plan.hpp): the CPU package (socket) of each worker core (parallel to+ /// `worker_cores`, -1: not known) and of the host core, the logical processors the process may use, whether the+ /// OS told physical cores from SMT siblings, and the container's CPU quota (0: none). On a multi-socket CPU the+ /// host's package comes first in `worker_cores`.+ std::vector<int> worker_package;+ int host_package = -1;+ int logical = 0;+ bool layout_known = true;+ double cpu_quota = 0;};+/// The description plan_pool_workers() takes, from a detected topology.+PoolPlanInput pool_plan_input(const CpuTopology& topo);+/// The CPUs a container quota allows (cgroup v2 cpu.max / v1 cfs quota), 0 when none or not Linux.+double container_cpu_quota();+/// Which core the host thread - the layer loop that spins on the GPU's flags - takes when `skip_first` reserves one/// (--host-core first|last|sibling, STRATA_HOST_CORE). `First` is the layout the pool has always used. `Last`/// (eddoursul's fork, F12) puts the host on the last physical core and lets the workers have the first: Windows sends@@ -165,6 +180,11 @@ public:ExpertPool& operator=(const ExpertPool&) = delete;int workers() const { return n_; }+ /// What bounded the default worker count (pool_plan.hpp; "" when a count was configured), the physical cores+ /// and logical processors counted - for the start-up line.+ const char* plan_limit() const { return plan_limit_; }+ int plan_cores() const { return plan_cores_; }+ int plan_logical() const { return plan_logical_; }/// Whether the host thread also drains. Reported at startup, because "the engine adapts to the machine it/// is on" is only true if the engine says which adaptation it took.bool host_works() const { return host_works_; }@@ -242,6 +262,9 @@ private:void publish();int n_ = 0;+ const char* plan_limit_ = "";+ int plan_cores_ = 0;+ int plan_logical_ = 0;bool host_works_ = true;ExpertJob* jobs_ = nullptr;int njobs_ = 0;diff --git a/include/strata/kernels/cpu/pool_plan.hpp b/include/strata/kernels/cpu/pool_plan.hppnew file mode 100644index 000000000..df91d6ea4--- /dev/null+++ b/include/strata/kernels/cpu/pool_plan.hpp@@ -0,0 +1,113 @@+// include/strata/kernels/cpu/pool_plan.hpp - how many CPU expert workers the engine starts by default, and when a+// configured count deserves a warning. Pure functions of a CPU description (no OS calls), so the choice is unit+// tested on mock topologies (pool_plan_test.cpp: 8c/16t, 16c/32t, 64c/128t, a 2-socket 128c/256t box, ...).+//+// Why a plan and not "every physical core but the host's":+// * the expert pool's workers spin while they wait. A worker without a core of its own (more workers than physical+// cores) takes cycles from a worker that has one, so a big count is slower than a small one, not just useless.+// A report from a 256-thread machine: ~256 workers, speed collapsed.+// * the kernel is DRAM-bound; past the memory system's width more cores add nothing, only the barrier at the end of+// every layer waits for more threads (kPoolWorkersCap, measured: phaseA-tests/notes-152-users.md).+// * on a 2-socket box the workers of the other socket read the experts across the interconnect (35 workers measured+// slower than 17-18 on 2-socket Xeons, bench #674 #707): the default stays on the host thread's socket.+// * a container limited to a few CPUs by a CFS quota (docker --cpus) still shows every core of the machine; spinning+// threads beyond the quota are throttled together.+//+// A count the user configures (--pool-workers) always runs as given; `pool_workers_warning` says in plain words when it+// is above what this CPU can use well (recommend, never force).+#pragma once++#include <algorithm>+#include <string>+#include <vector>++namespace strata::kernels::cpu {++/// The most default workers: past this the layer barrier costs more than the extra cores read (see the header comment).+/// Measured on 6c/12t (z440), 20c (P2003), 32c/64t (r9700) and 64c (l620): notes-152-users.md.+inline constexpr int kPoolWorkersCap = 24;+/// Cores kept free for the other threads of the engine (copy / IO threads, the GPU driver's, the OS) on a CPU with at+/// least `kPoolReserveFrom` cores that the pool may use (besides the host's). Small CPUs keep every core.+inline constexpr int kPoolReserveFrom = 16;+inline constexpr int kPoolReserve = 1;++struct PoolPlanInput {+ int logical = 0; ///< logical processors this process may run on+ /// physical cores this process may use per CPU package (socket), the host thread's package FIRST; the host+ /// thread's own core is counted in it. One entry on a single-socket CPU. Empty: the topology is not known.+ std::vector<int> cores_per_package;+ double cpu_quota = 0; ///< CPUs a container quota allows (cgroup cpu.max); 0: no quota+};++struct PoolPlan {+ int workers = 1; ///< the default count of pool workers (the host thread works beside them)+ int cores = 0; ///< physical cores counted (all packages)+ int cores_local = 0; ///< ... on the host's package+ int packages = 1;+ bool topology_known = true;+ const char* limit = ""; ///< what bounded the count (for the start-up line)+};++inline PoolPlan plan_pool_workers(const PoolPlanInput& in) {+ PoolPlan p;+ int cores = 0;+ for (int c : in.cores_per_package) cores += std::max(0, c);+ int local = in.cores_per_package.empty() ? 0 : std::max(0, in.cores_per_package.front());+ p.packages = std::max<int>(1, (int) in.cores_per_package.size());+ p.topology_known = !in.cores_per_package.empty() && cores > 0;+ if (!p.topology_known) {+ // The OS did not say which logical processors share a core. Counting every one as a core is what put a+ // 256-thread machine at 255 workers; a 2-way SMT is the usual case on a many-core CPU.+ cores = in.logical >= 16 ? in.logical / 2 : in.logical;+ local = cores;+ p.packages = 1;+ }+ p.cores = cores;+ p.cores_local = local;+ p.limit = "every physical core but the host's";+ int n = std::max(1, local - 1); // the host thread has the first core+ if (!p.topology_known) p.limit = "half the logical processors (the OS gave no core layout)";+ else if (p.packages > 1) p.limit = "the host's CPU socket";+ if (n >= kPoolReserveFrom) { // a big CPU keeps a core for the engine's other threads+ n -= kPoolReserve;+ if (p.topology_known && p.packages == 1) p.limit = "physical cores, one kept free";+ }+ if (in.cpu_quota > 0.0) {+ const int q = std::max(1, (int) in.cpu_quota - 1);+ if (q < n) {+ n = q;+ p.limit = "the container's CPU quota";+ }+ }+ if (n > kPoolWorkersCap) {+ n = kPoolWorkersCap;+ p.limit = "the measured cap";+ }+ p.workers = std::max(1, n);+ return p;+}++/// The warning for a configured worker count `requested` (> 0), or "" when it is fine. `plan` is plan_pool_workers(in).+inline std::string pool_workers_warning(int requested, const PoolPlanInput& in, const PoolPlan& plan) {+ if (requested <= 0 || requested <= plan.workers) return "";+ const std::string rec = "Recommended: --pool-workers " + std::to_string(plan.workers) ++ " (or leave it out: the engine picks that by itself). It still runs as configured.";+ const std::string asked = std::to_string(requested) + " CPU expert workers are configured";+ if (requested + 1 > in.logical && in.logical > 0)+ return asked + ", but this process can run only " + std::to_string(in.logical) ++ " CPU threads. The extra workers spin and take time from the others. " + rec;+ if (plan.topology_known && requested + 1 > plan.cores)+ return asked + ", but this CPU has " + std::to_string(plan.cores) ++ " physical cores. Two workers on one core share its memory bandwidth and slow each other down. " + rec;+ if (plan.packages > 1 && requested + 1 > plan.cores_local)+ return asked + ", more than the " + std::to_string(plan.cores_local) ++ " cores of the CPU socket the engine's main thread runs on. Workers on the other socket read the experts " ++ "across the link and were measured slower than fewer local ones. " + rec;+ if (in.cpu_quota > 0.0 && (double) requested + 1 > in.cpu_quota)+ return asked + ", but a container limit allows about " + std::to_string((int) in.cpu_quota) ++ " CPUs. Threads beyond the limit are throttled together. " + rec;+ return asked + ", more than this CPU can use well (the speed stops rising at about " ++ std::to_string(plan.workers) + " workers and the end of each layer waits for every one). " + rec;+}++} // namespace strata::kernels::cpudiff --git a/setup.py b/setup.pyindex 26457ae5d..5b454f325 100644--- a/setup.py+++ b/setup.py@@ -935,6 +935,7 @@ def recommend_pool_workers(args: list) -> list:n = hybrid_pool_workers(cpu_cores())if n is None or "--pool-workers" in args:return args+ n = min(n, POOL_WORKERS_CAP) # setup never writes a big worker count (see plan_pool_workers)p, e = cpu_cores()ok(f"hybrid CPU ({p} performance + {e} efficiency cores): {n} CPU expert workers - the performance cores and half ""of the efficiency cores (--pool-workers in the config; START-HERE --calibrate measures it on this PC)")@@ -993,6 +994,67 @@ def two_socket_note(sockets) -> list[str]:"the config's args (START-HERE --calibrate measures it on this PC)"]+# The engine's default count of CPU expert workers (include/strata/kernels/cpu/pool_plan.hpp, which tools/test_pool_plan.py+# keeps in step with these numbers). Setup writes no worker count on an ordinary CPU: the engine picks it, from the+# physical cores (not the SMT siblings), minus the host thread's core and a reserve on a big CPU, on the host thread's+# socket, under a cap. What setup does write (a hybrid CPU's count, the vision encoder's threads) is capped the same way.+POOL_WORKERS_CAP = 24+POOL_RESERVE_FROM = 16+POOL_RESERVE = 1+VISION_THREADS_CAP = 16+++def plan_pool_workers(logical: int, cores_per_package=None, quota: float = 0.0) -> int:+ """The engine's default pool worker count for a CPU of `logical` threads whose physical cores per socket are+ `cores_per_package` (the host thread's socket first; None/empty: the layout is not known) and a container CPU+ quota (0: none)."""+ cores = list(cores_per_package or [])+ if cores and sum(cores) > 0:+ local = cores[0]+ else: # no layout: half the logical processors on a many-thread CPU+ local = logical // 2 if logical >= 16 else logical+ n = max(1, local - 1) # the host thread has the first core+ if n >= POOL_RESERVE_FROM:+ n -= POOL_RESERVE+ if quota > 0:+ n = min(n, max(1, int(quota) - 1))+ return max(1, min(n, POOL_WORKERS_CAP))+++def cpu_core_layout():+ """(logical threads, [physical cores per socket]); the cores list is empty when the OS does not say."""+ logical = os.cpu_count() or 1+ sockets = cpu_sockets()+ return logical, ([sockets[1]] * sockets[0] if sockets else [])+++def pool_workers_note(args, layout=None) -> list[str]:+ """A plain warning when the config's `--pool-workers` is above what this CPU uses well. Recommend, never force: the+ count stays as it is and the engine runs with it."""+ if "--pool-workers" not in args:+ return []+ try:+ asked = int(args[args.index("--pool-workers") + 1])+ except (IndexError, ValueError):+ return []+ logical, cores = layout or cpu_core_layout()+ rec = plan_pool_workers(logical, cores)+ if asked <= rec:+ return []+ why = ("more than this PC has CPU threads" if asked + 1 > logical else+ "more than its physical cores" if cores and asked + 1 > sum(cores) else+ "more than the cores of one CPU socket" if len(cores) > 1 and asked + 1 > cores[0] else+ "more than the memory bandwidth can feed")+ return [f"the config asks for {asked} CPU expert workers, {why}: extra workers can make Strata slower. Recommended: "+ f"--pool-workers {rec} (or remove the setting: the engine picks that by itself). It stays as you set it."]+++def vision_threads(logical: int | None = None) -> int:+ """The CPU vision encoder's threads: one per physical core as before (half the logical threads), at most+ VISION_THREADS_CAP - a 256-thread server does not need 128 of them."""+ return max(1, min((logical if logical is not None else (os.cpu_count() or 8)) // 2, VISION_THREADS_CAP))++def cpu_info():"""(name, avx2, avx512): avx512 means everything Strata's fast AVX-512 kernels use (F, BW, VL, VNNI, VBMI),the same test the engine makes (cpu_avx512_ok), not just AVX-512F."""@@ -6180,6 +6242,8 @@ def install_own_path(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adopteif a.browser is not None:cfg["open_browser"] = a.browsercfg["args"] = recommend_pool_workers(cfg["args"]) # --pool-workers: this engine path's CPU expert threads too+ for line in pool_workers_note(cfg["args"]):+ warn(line)cfg_path = home() / f"strata-{tag.lower()}.json"if vision != "none":old_cfg = cfg_path if cfg_path.is_file() else adopted@@ -6188,7 +6252,7 @@ def install_own_path(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adoptecfg["vision"] = {"exe": str(eng / VEXE), "mmproj": str(mmproj), "model": str(gguf), "gpu": vision == "gpu","max_tokens": vt} # the text model, opened vocab-onlyif vision == "cpu":- cfg["vision"]["threads"] = max(1, (os.cpu_count() or 8) // 2)+ cfg["vision"]["threads"] = vision_threads()elif getattr(a, "vision_tokens", None) is not None:warn("--vision-tokens: images are off, so it is not used")# requests at once ("parallel", opt-in): written only when given; the recommendation is said once@@ -7577,7 +7641,7 @@ def main() -> int:"model": str(ready_paths(model, models_dir)["dense"] if ready else shards[0]),"gpu": vision == "gpu", "max_tokens": vt}if vision == "cpu":- cfg["vision"]["threads"] = max(1, (os.cpu_count() or 8) // 2)+ cfg["vision"]["threads"] = vision_threads()elif a.vision_tokens is not None:warn("--vision-tokens: images are off for this model, so it is not used")if a.images: # only when given: else an earlier "images" is carried over@@ -7593,6 +7657,8 @@ def main() -> int:ok("the settings tuned for this PC earlier are used" + (f" ({cal['date']})" if cal.get("date") else ""))else: # #642: measured counts (a calibration) win over the rulecfg["args"] = recommend_pool_workers(cfg["args"])+ for line in pool_workers_note(cfg["args"]):+ warn(line)for line in two_socket_note(cpu_sockets()): # bench tips: text only, the config is not touchedsay(" " + line)for line in bench_tips(cfg["args"], cfg.get("env"), ram, MODELS[model]["ram_gb"], gpu.get("vram_gb", 0.0), vision, WIN):diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cppindex d9961938a..6501fae58 100644--- a/src/kernels/cpu/pool.cpp+++ b/src/kernels/cpu/pool.cpp@@ -15,7 +15,10 @@#include <cstdio>#include <cstdlib>+#include <cstring>#include <stdexcept>+#include <string>+#include <thread>#if defined(_WIN32)#define WIN32_LEAN_AND_MEAN@@ -79,6 +82,59 @@ HostCore host_core_setting() { return (HostCore) g_host_core.load(); }static CpuTopology detect_cpu_topology_impl(bool skip_first, PoolAffinity affinity);+// The fields the default worker count needs (pool_plan.hpp): the logical processors the process may use, the CPU+// package of the host core and of every worker core, whether the OS told physical cores from SMT siblings, the+// container CPU quota. On a multi-socket CPU (not hybrid) the host package comes first in `worker_cores`.+static void fill_plan_fields(CpuTopology& topo, bool skip_first) {+ topo.cpu_quota = container_cpu_quota();+ topo.worker_package.assign(topo.worker_cores.size(), -1);+#if defined(_WIN32)+ (void) skip_first;+ topo.logical = (int) GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);+ return; // (the sockets of a Windows box are not told apart: one package)+#elif defined(__APPLE__)+ (void) skip_first;+ topo.logical = (int) std::thread::hardware_concurrency();+ return; // no SMT on Apple silicon and no sysfs: the cores are the processors+#else+ std::vector<unsigned long> mask;+ std::vector<int> allowed;+ topo.logical = detail::get_thread_affinity(mask, &allowed) == 0 ? (int) allowed.size()+ : (int) std::thread::hardware_concurrency();+ auto read = [](int cpu, const char* what) -> long {+ char path[96];+ std::snprintf(path, sizeof path, "/sys/devices/system/cpu/cpu%d/topology/%s", cpu, what);+ long v = -1;+ if (std::FILE* f = std::fopen(path, "r")) {+ if (std::fscanf(f, "%ld", &v) != 1) v = -1;+ std::fclose(f);+ }+ return v;+ };+ bool known = !topo.worker_cores.empty() || topo.host_core >= 0;+ auto pkg_of = [&](int cpu) {+ const long pkg = read(cpu, "physical_package_id"), core = read(cpu, "core_id");+ if (pkg < 0 || core < 0) known = false;+ return (int) pkg;+ };+ if (topo.host_core >= 0) topo.host_package = pkg_of(topo.host_core);+ for (size_t i = 0; i < topo.worker_cores.size(); ++i) topo.worker_package[i] = pkg_of(topo.worker_cores[i]);+ topo.layout_known = known;+ if (!known) return;+ if (skip_first && !topo.is_hybrid && topo.host_core >= 0 && topo.host_package >= 0) {+ // the host socket first (stable inside each socket): the default workers are the local ones+ std::vector<size_t> order(topo.worker_cores.size());+ for (size_t i = 0; i < order.size(); ++i) order[i] = i;+ std::stable_partition(order.begin(), order.end(),+ [&](size_t i) { return topo.worker_package[i] == topo.host_package; });+ std::vector<int> cores, pkgs;+ for (size_t i : order) { cores.push_back(topo.worker_cores[i]); pkgs.push_back(topo.worker_package[i]); }+ topo.worker_cores = std::move(cores);+ topo.worker_package = std::move(pkgs);+ }+#endif+}+CpuTopology detect_cpu_topology(bool skip_first, PoolAffinity affinity) {CpuTopology topo = detect_cpu_topology_impl(skip_first, affinity);// --host-core last: the host takes the last core, the workers the others (the first included)@@ -96,9 +152,60 @@ CpuTopology detect_cpu_topology(bool skip_first, PoolAffinity affinity) {topo.worker_cores.end());topo.host_core = topo.host_sibling;}+ fill_plan_fields(topo, skip_first);return topo;}+double container_cpu_quota() {+#if defined(__linux__)+ // cgroup v2: "max 100000" (no limit) or "<quota> <period>"; v1: cpu.cfs_quota_us (-1: none) / cpu.cfs_period_us+ if (std::FILE* f = std::fopen("/sys/fs/cgroup/cpu.max", "r")) {+ char a[32] = {0};+ long period = 0;+ double q = 0;+ if (std::fscanf(f, "%31s %ld", a, &period) == 2 && std::strcmp(a, "max") != 0 && period > 0)+ q = std::atof(a) / (double) period;+ std::fclose(f);+ return q > 0 ? q : 0.0;+ }+ long quota = -1, period = 0;+ if (std::FILE* f = std::fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r")) {+ if (std::fscanf(f, "%ld", "a) != 1) quota = -1;+ std::fclose(f);+ }+ if (std::FILE* f = std::fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r")) {+ if (std::fscanf(f, "%ld", &period) != 1) period = 0;+ std::fclose(f);+ }+ return quota > 0 && period > 0 ? (double) quota / (double) period : 0.0;+#else+ return 0.0;+#endif+}++PoolPlanInput pool_plan_input(const CpuTopology& topo) {+ PoolPlanInput in;+ in.logical = topo.logical > 0 ? topo.logical : (int) std::thread::hardware_concurrency();+ in.cpu_quota = topo.cpu_quota;+ if (!topo.layout_known) return in; // cores_per_package stays empty+ // the cores per package, the host package first; the host core counts in its package+ std::vector<std::pair<int, int>> counts; // (package, cores)+ auto add = [&](int pkg) {+ for (auto& c : counts)+ if (c.first == pkg) { ++c.second; return; }+ counts.emplace_back(pkg, 1);+ };+ if (topo.host_core >= 0) add(topo.host_package);+ for (size_t i = 0; i < topo.worker_cores.size(); ++i)+ add(i < topo.worker_package.size() ? topo.worker_package[i] : -1);+ if (topo.host_core >= 0) { // (the host package first even when it has no worker core listed)+ auto it = std::find_if(counts.begin(), counts.end(), [&](const auto& c) { return c.first == topo.host_package; });+ if (it != counts.begin()) std::iter_swap(counts.begin(), it);+ }+ for (const auto& c : counts) in.cores_per_package.push_back(c.second);+ return in;+}+int planned_host_core(PoolAffinity affinity) { return detect_cpu_topology(true, affinity).host_core; }static CpuTopology detect_cpu_topology_impl(bool skip_first, PoolAffinity affinity) {@@ -501,13 +608,22 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity afif (tasks < 0 || tasks > kMaxTasks) throw std::invalid_argument("expert pool tasks must be in 0..4096");if (const char* e = std::getenv("STRATA_POOL_SPIN_US")) // a test knob; see kSpinBeforeSleepspin_before_sleep_ = std::chrono::microseconds((std::max)(0, std::atoi(e)));+ const PoolPlanInput plan_in = pool_plan_input(topo_);+ const PoolPlan plan = plan_pool_workers(plan_in);if (n_workers > 0) {n_ = n_workers;+ // recommend, never force: a count above what this CPU uses well runs as configured, with a plain warning+ static std::atomic<bool> warned{false};+ const std::string why = pool_workers_warning(n_workers, plan_in, plan);+ if (!why.empty() && !warned.exchange(true)) std::fprintf(stderr, "strata cpu pool: warning: %s\n", why.c_str());} else if (topo_.is_hybrid && affinity_ != PoolAffinity::All) {n_ = (std::max)(1, topo_.p_cores - 1);} else {- n_ = (int) topo_.worker_cores.size();+ n_ = (std::min)(plan.workers, (std::max)(1, (int) topo_.worker_cores.size()));+ plan_limit_ = plan.limit;}+ plan_cores_ = plan.cores;+ plan_logical_ = plan_in.logical;if (n_ < 1) n_ = 1;scratch_.resize((size_t) n_);wstate_.reset(new std::atomic<int32_t>[(size_t) n_]);diff --git a/src/kernels/cpu/pool_plan_test.cpp b/src/kernels/cpu/pool_plan_test.cppnew file mode 100644index 000000000..84fb63098--- /dev/null+++ b/src/kernels/cpu/pool_plan_test.cpp@@ -0,0 +1,119 @@+// pool_plan_test: the default CPU expert worker count on mock CPUs (include/strata/kernels/cpu/pool_plan.hpp).+// Pure arithmetic, no threads and no OS calls: it runs on any machine.+#include "strata/kernels/cpu/pool_plan.hpp"++#include <cstdio>+#include <string>++namespace cpu = strata::kernels::cpu;++static int g_fail = 0;++static void expect(bool ok, const char* what) {+ if (!ok) {+ ++g_fail;+ std::fprintf(stderr, "FAIL: %s\n", what);+ }+}++static cpu::PoolPlanInput cpu_of(int logical, std::vector<int> per_package, double quota = 0) {+ cpu::PoolPlanInput in;+ in.logical = logical;+ in.cores_per_package = std::move(per_package);+ in.cpu_quota = quota;+ return in;+}++static void line(const char* name, const cpu::PoolPlanInput& in) {+ const cpu::PoolPlan p = cpu::plan_pool_workers(in);+ std::printf(" %-34s logical %3d cores %3d (host socket %3d, %d socket%s) -> %2d workers [%s]\n", name, in.logical,+ p.cores, p.cores_local, p.packages, p.packages == 1 ? "" : "s", p.workers, p.limit);+}++int main() {+ std::printf("pool_plan_test: cap %d, reserve %d from %d workers\n", cpu::kPoolWorkersCap, cpu::kPoolReserve,+ cpu::kPoolReserveFrom);++ // 8 cores / 16 threads: every physical core but the host's, never the SMT siblings+ const auto c8 = cpu_of(16, {8});+ line("8c/16t", c8);+ expect(cpu::plan_pool_workers(c8).workers == 7, "8c/16t -> 7 workers");++ // 6 cores / 12 threads (the small test PCs) and a CPU without SMT+ expect(cpu::plan_pool_workers(cpu_of(12, {6})).workers == 5, "6c/12t -> 5 workers");+ expect(cpu::plan_pool_workers(cpu_of(4, {4})).workers == 3, "4c/4t (no SMT) -> 3 workers");+ expect(cpu::plan_pool_workers(cpu_of(1, {1})).workers == 1, "1 core -> 1 worker (never 0)");+ expect(cpu::plan_pool_workers(cpu_of(2, {2})).workers == 1, "2 cores -> 1 worker");++ // 16 cores / 32 threads+ const auto c16 = cpu_of(32, {16});+ line("16c/32t", c16);+ {+ const int w = cpu::plan_pool_workers(c16).workers;+ expect(w == 15 - (15 >= cpu::kPoolReserveFrom ? cpu::kPoolReserve : 0) || w == cpu::kPoolWorkersCap,+ "16c/32t -> physical cores minus the host (and the reserve)");+ expect(w <= 15, "16c/32t never above the physical cores minus the host");+ }++ // 64 cores / 128 threads: the cap, far below the logical count+ const auto c64 = cpu_of(128, {64});+ line("64c/128t", c64);+ expect(cpu::plan_pool_workers(c64).workers == cpu::kPoolWorkersCap, "64c/128t -> the cap");+ expect(cpu::plan_pool_workers(c64).workers < 64, "64c/128t is not near the logical count");++ // 2 sockets x 64 cores / 256 threads: the user report - never 255, one socket's cores at most, under the cap+ const auto c2s = cpu_of(256, {64, 64});+ line("2-socket 128c/256t", c2s);+ expect(cpu::plan_pool_workers(c2s).workers == cpu::kPoolWorkersCap, "2-socket 128c/256t -> the cap");+ expect(cpu::plan_pool_workers(c2s).packages == 2, "2-socket: two packages");+ expect(cpu::plan_pool_workers(c2s).cores == 128, "2-socket: 128 physical cores counted");++ // 2 sockets x 20 cores (the Xeons of bench #674 #707: 17-18 workers beat 35): the host's socket only+ const auto xeon = cpu_of(80, {20, 20});+ line("2-socket 40c/80t (Xeon)", xeon);+ expect(cpu::plan_pool_workers(xeon).workers == 18, "2x20c: 18 workers (bench #674 #707: 17-18 beat 35): the host socket only, minus the host and the reserve");+ // ... and a smaller second socket does not matter; the host's socket comes first in the input+ expect(cpu::plan_pool_workers(cpu_of(48, {6, 18})).workers == 5, "host socket of 6 cores: 5 workers");++ // an unknown core layout (no sysfs, a container): half the logical processors for a many-thread CPU, never all+ const auto unk = cpu_of(256, {});+ line("256 logical, layout unknown", unk);+ expect(!cpu::plan_pool_workers(unk).topology_known, "unknown layout is flagged");+ expect(cpu::plan_pool_workers(unk).workers <= cpu::kPoolWorkersCap, "unknown 256 -> at most the cap");+ expect(cpu::plan_pool_workers(unk).workers < 255, "unknown 256 -> never 255");+ expect(cpu::plan_pool_workers(cpu_of(8, {})).workers == 7, "unknown, 8 logical -> 7 workers");++ // a container quota+ const auto q = cpu_of(128, {64}, 4.0);+ line("64c, container quota 4 CPUs", q);+ expect(cpu::plan_pool_workers(q).workers == 3, "quota 4 CPUs -> 3 workers");+ expect(cpu::plan_pool_workers(cpu_of(128, {64}, 200.0)).workers == cpu::kPoolWorkersCap, "a big quota changes nothing");++ // warnings: only above the plan, and each says what to do+ const cpu::PoolPlan p2s = cpu::plan_pool_workers(c2s);+ expect(cpu::pool_workers_warning(0, c2s, p2s).empty(), "0 (the default) never warns");+ expect(cpu::pool_workers_warning(p2s.workers, c2s, p2s).empty(), "the plan itself never warns");+ expect(cpu::pool_workers_warning(4, c2s, p2s).empty(), "fewer than the plan never warns");+ {+ const std::string w = cpu::pool_workers_warning(256, c2s, p2s);+ expect(!w.empty(), "256 workers on 256 threads warns");+ expect(w.find("--pool-workers " + std::to_string(p2s.workers)) != std::string::npos, "the warning names the recommendation");+ expect(w.find("still runs") != std::string::npos, "the warning says it still runs");+ }+ expect(cpu::pool_workers_warning(200, c2s, p2s).find("physical cores") != std::string::npos,+ "200 workers on 128 cores: more than the physical cores");+ expect(cpu::pool_workers_warning(300, c2s, p2s).find("256 CPU threads") != std::string::npos,+ "300 workers on 256 threads: more than the CPU threads");+ expect(cpu::pool_workers_warning(30, xeon, cpu::plan_pool_workers(xeon)).find("socket") != std::string::npos,+ "30 workers on 2x20c: crosses the socket");+ expect(!cpu::pool_workers_warning(30, c64, cpu::plan_pool_workers(c64)).empty(), "30 workers on 64c is above the cap: warns");+ expect(cpu::pool_workers_warning(8, q, cpu::plan_pool_workers(q)).find("container") != std::string::npos,+ "8 workers under a 4-CPU quota: names the container limit");++ if (g_fail) {+ std::fprintf(stderr, "pool_plan_test: %d failure(s)\n", g_fail);+ return 1;+ }+ std::printf("pool_plan_test: OK\n");+ return 0;+}diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 5b299bd81..679fcfdfb 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -5691,6 +5691,9 @@ int main(int argc, char** argv) {o.pool_tasks ? o.pool_tasks : 3 * (pool.workers() + (pool.host_works() ? 1 : 0)),o.pool_tasks ? " (capped by rows)" : " (automatic)",pool.workers() + (pool.host_works() ? 1 : 0));+ if (pool.plan_limit()[0] != '\0') // the default count: what it was bounded by (a configured count is never changed)+ std::fprintf(stderr, "strata generate: CPU pool: %d workers by default (%d physical cores, %d logical processors; bounded by %s)\n",+ pool.workers(), pool.plan_cores(), pool.plan_logical(), pool.plan_limit());if (pool.is_hybrid() && pool.affinity() != strata::kernels::cpu::PoolAffinity::All) {const char* aff_str = pool.affinity() == strata::kernels::cpu::PoolAffinity::PCores ? "p-cores" :pool.affinity() == strata::kernels::cpu::PoolAffinity::All ? "all" : "auto";diff --git a/tools/test_pool_plan.py b/tools/test_pool_plan.pynew file mode 100644index 000000000..ea69df4a6--- /dev/null+++ b/tools/test_pool_plan.py@@ -0,0 +1,105 @@+"""The default count of CPU expert workers on mock CPUs, and what setup writes about thread counts.++The engine's choice is include/strata/kernels/cpu/pool_plan.hpp (tested on its own by the pool_plan_test binary); setup mirrors+it so it can warn about a configured count. This test keeps the two in step and checks the mirror on the same mock+topologies (8c/16t, 16c/32t, 64c/128t, a 2-socket 128c/256t box) plus the reports behind it: a 256-thread machine started+~256 workers and slowed down.++ python -m unittest tools.test_pool_plan+"""+from __future__ import annotations++import re+import sys+import unittest+from pathlib import Path+from unittest import mock++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))+sys.path.insert(0, str(ROOT / "tools"))+import setup # noqa: E402++HEADER = ROOT / "include" / "strata" / "kernels" / "cpu" / "pool_plan.hpp"+++class InStepWithTheEngine(unittest.TestCase):+ def header_constant(self, name):+ m = re.search(rf"constexpr int {name}\s*=\s*(\d+);", HEADER.read_text(encoding="utf-8"))+ self.assertIsNotNone(m, name)+ return int(m.group(1))++ def test_constants(self):+ self.assertEqual(setup.POOL_WORKERS_CAP, self.header_constant("kPoolWorkersCap"))+ self.assertEqual(setup.POOL_RESERVE_FROM, self.header_constant("kPoolReserveFrom"))+ self.assertEqual(setup.POOL_RESERVE, self.header_constant("kPoolReserve"))+++class DefaultWorkers(unittest.TestCase):+ def test_mock_topologies(self):+ cap = setup.POOL_WORKERS_CAP+ # physical cores, not threads, minus the host thread's core+ self.assertEqual(setup.plan_pool_workers(16, [8]), 7) # 8c/16t+ self.assertEqual(setup.plan_pool_workers(12, [6]), 5) # 6c/12t+ self.assertEqual(setup.plan_pool_workers(32, [16]), 15) # 16c/32t+ self.assertEqual(setup.plan_pool_workers(128, [64]), cap) # 64c/128t: the cap+ self.assertEqual(setup.plan_pool_workers(256, [64, 64]), cap) # 2-socket 128c/256t: the cap, never 255+ self.assertLess(setup.plan_pool_workers(256, [64, 64]), 128)+ # two sockets of 20 cores: the host's socket only (bench #674 #707: 17-18 workers beat 35)+ self.assertEqual(setup.plan_pool_workers(80, [20, 20]), 18)+ self.assertEqual(setup.plan_pool_workers(48, [6, 18]), 5)+ self.assertEqual(setup.plan_pool_workers(1, [1]), 1)++ def test_unknown_layout_and_quota(self):+ self.assertLessEqual(setup.plan_pool_workers(256, []), setup.POOL_WORKERS_CAP)+ self.assertLess(setup.plan_pool_workers(256, None), 255)+ self.assertEqual(setup.plan_pool_workers(8, None), 7)+ self.assertEqual(setup.plan_pool_workers(128, [64], quota=4.0), 3) # a container limit of 4 CPUs++ def test_never_writes_a_big_count(self):+ # the hybrid rule (P-cores + half the E-cores) is capped like every other count+ with mock.patch.object(setup, "cpu_cores", return_value=(8, 120)):+ args = setup.recommend_pool_workers(["--spec", "4"])+ n = int(args[args.index("--pool-workers") + 1])+ self.assertLessEqual(n, setup.POOL_WORKERS_CAP)+ with mock.patch.object(setup, "cpu_cores", return_value=(8, 16)): # an ordinary hybrid: the usual 15+ args = setup.recommend_pool_workers([])+ self.assertEqual(args, ["--pool-workers", "15"])+ with mock.patch.object(setup, "cpu_cores", return_value=None): # not hybrid: nothing written+ self.assertEqual(setup.recommend_pool_workers(["--spec", "4"]), ["--spec", "4"])+ # a count that is already there is never changed+ with mock.patch.object(setup, "cpu_cores", return_value=(8, 120)):+ self.assertEqual(setup.recommend_pool_workers(["--pool-workers", "200"]), ["--pool-workers", "200"])++ def test_vision_threads(self):+ self.assertEqual(setup.vision_threads(16), 8)+ self.assertEqual(setup.vision_threads(256), setup.VISION_THREADS_CAP)+ self.assertEqual(setup.vision_threads(1), 1)+++class ConfiguredCountWarns(unittest.TestCase):+ big = (256, [64, 64])++ def test_warns_above_the_plan_and_keeps_the_count(self):+ args = ["--pool-workers", "255"]+ note = setup.pool_workers_note(args, self.big)+ self.assertEqual(len(note), 1)+ self.assertIn("--pool-workers %d" % setup.plan_pool_workers(*self.big), note[0])+ self.assertIn("stays as you set it", note[0])+ self.assertEqual(args, ["--pool-workers", "255"]) # nothing rewritten++ def test_quiet_when_fine(self):+ self.assertEqual(setup.pool_workers_note([], self.big), [])+ self.assertEqual(setup.pool_workers_note(["--pool-workers", "8"], self.big), [])+ self.assertEqual(setup.pool_workers_note(["--pool-workers", str(setup.plan_pool_workers(*self.big))], self.big), [])+ self.assertEqual(setup.pool_workers_note(["--pool-workers"], self.big), [])+ self.assertEqual(setup.pool_workers_note(["--pool-workers", "x"], self.big), [])++ def test_reasons(self):+ self.assertIn("CPU threads", setup.pool_workers_note(["--pool-workers", "300"], self.big)[0])+ self.assertIn("physical cores", setup.pool_workers_note(["--pool-workers", "200"], self.big)[0])+ self.assertIn("one CPU socket", setup.pool_workers_note(["--pool-workers", "30"], (80, [20, 20]))[0])+++if __name__ == "__main__":+ unittest.main()diff --git a/tools/vision/strata_vision.cpp b/tools/vision/strata_vision.cppindex 9a06ce1fa..556802fa4 100644--- a/tools/vision/strata_vision.cpp+++ b/tools/vision/strata_vision.cpp@@ -129,7 +129,8 @@ int main(int argc, char** argv) {if (!gpu && !fa_given && ggml_cpu_has_avx512()) fa = LLAMA_FLASH_ATTN_TYPE_DISABLED;cp.flash_attn_type = fa;// on the CPU without --threads: one per core (mtmd's own default is 4 threads)- if (threads <= 0 && !gpu) threads = std::max(1u, std::thread::hardware_concurrency() / 2);+ // (at most 16: a 256-thread server does not need 128 encoder threads; --threads overrides)+ if (threads <= 0 && !gpu) threads = std::max(1u, std::min(16u, std::thread::hardware_concurrency() / 2));if (threads > 0) cp.n_threads = threads;if (max_tokens > 0) cp.image_max_tokens = max_tokens;if (min_tokens > 0) cp.image_min_tokens = min_tokens;