Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
17.6 KB
Raw Blame
// strata/platform/aux_cpus.hpp - where the engine's OTHER threads run.
//
// The pool pins its workers (one logical CPU per physical core) and the host thread pins itself to the pool's reserved
// core.  Every thread the host creates afterwards inherits the host's single CPU (the adaptive tier's job thread, the
// prefill helpers, the router lookahead ...), and the threads the CUDA driver creates float over every CPU, the pool
// workers' included.  Measured on a 16-worker box: while the adaptive tier was busy the host thread was preempted 5x
// as often and the per-layer wait for the host's flag was 60-155 us instead of 8.
//
// `--aux-cpus LIST|auto|off` (STRATA_AUX_CPUS) puts every thread that is neither a pool worker nor the host thread on
// a small set of CPUs no worker uses:
//   * threads the engine creates call `pin_current_thread()` first thing (a no-op while the feature is off);
//   * threads somebody else created (the CUDA driver's) are caught by `sweep()` over /proc/self/task, run after the
//     start-up and at every request.
// `auto` takes the SMT siblings of the host's core, then every CPU of a physical core no worker is pinned to.  It
// never takes a CPU that shares a core with a worker (that would steal the worker's cycles): with nothing spare the
// feature does nothing.  A list (`--aux-cpus 24,26`) is taken as given, minus the host's CPU and the workers' CPUs.
//
// Linux only (the other systems get empty functions).  Header-only on purpose: the callers sit in four libraries that
// do not link each other.  No output of the engine depends on where a thread runs.
#pragma once

#include "strata/platform/thread_name.hpp"
#include <algorithm>
#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
#include <mutex>
#include <set>
#include <string>
#include <vector>

#if defined(__linux__)
#include <dirent.h>
#include <sched.h>
#include <sys/syscall.h>
#include <unistd.h>
#endif

namespace strata::aux_cpus {

/// One logical CPU the process may use and the physical core it belongs to (-1: unknown).
struct LogicalCpu {
    int cpu = -1;
    long pkg = -1;
    long core = -1;
};

/// "24,26,30-31" -> {24, 26, 30, 31} (sorted, no duplicates).  False for anything else (empty items, letters,
/// reversed ranges, negative numbers); `out` is then empty.
inline bool parse_cpu_list(const std::string& text, std::vector<int>& out) {
    out.clear();
    if (text.empty()) return false;
    size_t i = 0;
    auto number = [&](long& v) {
        if (i >= text.size() || text[i] < '0' || text[i] > '9') return false;
        v = 0;
        while (i < text.size() && text[i] >= '0' && text[i] <= '9') {
            v = v * 10 + (text[i] - '0');
            if (v > 100000) return false;
            ++i;
        }
        return true;
    };
    for (;;) {
        long lo = 0, hi = 0;
        if (!number(lo)) { out.clear(); return false; }
        hi = lo;
        if (i < text.size() && text[i] == '-') {
            ++i;
            if (!number(hi) || hi < lo) { out.clear(); return false; }
        }
        for (long c = lo; c <= hi; ++c) out.push_back((int) c);
        if (i == text.size()) break;
        if (text[i] != ',') { out.clear(); return false; }
        ++i;
    }
    std::sort(out.begin(), out.end());
    out.erase(std::unique(out.begin(), out.end()), out.end());
    return true;
}

/// The CPUs `auto` gives to the aux threads, in order: the SMT siblings of the host's physical core first, then every
/// CPU of a physical core that has no worker and is not the host's.  A CPU that shares a core with a worker is never
/// returned, and neither is one whose core is unknown (no sysfs): it might be a worker's sibling.
inline std::vector<int> derive(const std::vector<LogicalCpu>& allowed, int host_cpu, const std::vector<int>& worker_cpus) {
    using Key = std::pair<long, long>;
    auto key_of = [&](int cpu, Key& k) {
        for (const LogicalCpu& c : allowed)
            if (c.cpu == cpu) { k = {c.pkg, c.core}; return c.pkg >= 0 && c.core >= 0; }
        return false;
    };
    std::vector<Key> worker_keys;
    for (int w : worker_cpus) {
        Key k;
        if (key_of(w, k)) worker_keys.push_back(k);
    }
    Key host_key;
    const bool host_known = host_cpu >= 0 && key_of(host_cpu, host_key);
    std::vector<int> siblings, free_cores;
    for (const LogicalCpu& c : allowed) {
        if (c.cpu == host_cpu || c.pkg < 0 || c.core < 0) continue;
        if (std::find(worker_cpus.begin(), worker_cpus.end(), c.cpu) != worker_cpus.end()) continue;
        const Key k{c.pkg, c.core};
        if (host_known && k == host_key) {
            siblings.push_back(c.cpu);
        } else if (std::find(worker_keys.begin(), worker_keys.end(), k) == worker_keys.end()) {
            free_cores.push_back(c.cpu);
        }
    }
    std::sort(siblings.begin(), siblings.end());
    std::sort(free_cores.begin(), free_cores.end());
    siblings.insert(siblings.end(), free_cores.begin(), free_cores.end());
    return siblings;
}

/// An explicit list, restricted to CPUs the process may use, without the host's CPU and the workers' CPUs.
inline std::vector<int> restrict_list(const std::vector<int>& list, const std::vector<LogicalCpu>& allowed, int host_cpu,
                                      const std::vector<int>& worker_cpus) {
    std::vector<int> out;
    for (int c : list) {
        if (c == host_cpu || std::find(worker_cpus.begin(), worker_cpus.end(), c) != worker_cpus.end()) continue;
        const bool ok = std::any_of(allowed.begin(), allowed.end(), [&](const LogicalCpu& a) { return a.cpu == c; });
        if (ok) out.push_back(c);
    }
    return out;
}

/// What `--aux-cpus` / STRATA_AUX_CPUS said.  `on`: placed from the start-up; `use_list`: the list, not `auto`.
struct Config {
    bool on = false;
    bool use_list = false;
    std::vector<int> list;
};

/// "" and "off": off at start (a request can still turn it on with `auto`); "auto"; or a CPU list.
inline bool parse_spec(const std::string& spec, Config& cfg, std::string& err) {
    cfg = Config{};
    if (spec.empty() || spec == "off") return true;
    if (spec == "auto") { cfg.on = true; return true; }
    if (!parse_cpu_list(spec, cfg.list)) {
        err = "--aux-cpus takes off, auto or a CPU list such as 24,26 or 24-27 (got '" + spec + "')";
        return false;
    }
    cfg.on = true;
    cfg.use_list = true;
    return true;
}

#if defined(__linux__)

namespace detail {

struct State {
    std::mutex mu;
    Config cfg;
    std::atomic<bool> enabled{false};
    std::atomic<bool> have_plan{false};
    std::vector<int> allowed;                     ///< the process's CPUs, snapshotted before any thread is pinned
    std::vector<int> plan;                        ///< the aux CPUs (empty: nothing to do)
    std::vector<int> workers;                     ///< the pool's CPUs: their threads are never touched
    int host_cpu = -1;                            ///< the CPU the host thread is pinned to (-1: unknown)
    cpu_set_t target;
    std::set<int> owned;                          ///< tids of pool workers and host threads
    std::map<int, cpu_set_t> original;            ///< tid -> the mask it had before this module moved it
    State() { CPU_ZERO(&target); }
};

inline State& state() {
    static State s;
    return s;
}

inline int self_tid() { return (int) syscall(SYS_gettid); }

inline bool same_mask(const cpu_set_t& a, const cpu_set_t& b) { return CPU_EQUAL(&a, &b) != 0; }

/// Moves `tid` to the aux set.  Caller holds `mu`.  Returns true if the thread's mask changed.
inline bool move_locked(State& s, int tid) {
    cpu_set_t now;
    CPU_ZERO(&now);
    if (sched_getaffinity(tid, sizeof now, &now) != 0) return false;
    if (same_mask(now, s.target)) return false;
    if (CPU_COUNT(&now) == 1) {                    // pinned to one CPU of a worker by somebody: not ours
        for (int w : s.workers)
            if (CPU_ISSET(w, &now)) return false;
    }
    if (sched_setaffinity(tid, sizeof s.target, &s.target) != 0) return false;
    s.original.emplace(tid, now);                 // the first mask only: a second move keeps the real original
    return true;
}

inline bool alive(int tid) {
    char path[48];
    std::snprintf(path, sizeof path, "/proc/self/task/%d", tid);
    return access(path, F_OK) == 0;
}

inline std::vector<int> task_ids() {
    std::vector<int> ids;
    if (DIR* d = opendir("/proc/self/task")) {
        while (const dirent* e = readdir(d))
            if (e->d_name[0] >= '0' && e->d_name[0] <= '9') ids.push_back(std::atoi(e->d_name));
        closedir(d);
    }
    return ids;
}

/// Forgets the threads that have ended (their tids may come back for another thread).  Caller holds `mu`.
inline void purge_locked(State& s) {
    for (auto it = s.original.begin(); it != s.original.end();) it = alive(it->first) ? std::next(it) : s.original.erase(it);
    for (auto it = s.owned.begin(); it != s.owned.end();) it = alive(*it) ? std::next(it) : s.owned.erase(it);
}

}  // namespace detail

/// Reads the CPUs the calling thread may use and the spec.  Call once, early, on the thread that has not been pinned
/// yet (the engine's main thread, before the pool exists).  The main thread counts as the host.
inline bool configure(const std::string& spec, std::string& err) {
    detail::State& s = detail::state();
    Config cfg;
    if (!parse_spec(spec, cfg, err)) return false;
    std::lock_guard<std::mutex> lk(s.mu);
    s.cfg = cfg;
    s.allowed.clear();
    cpu_set_t now;
    CPU_ZERO(&now);
    if (sched_getaffinity(0, sizeof now, &now) == 0)
        for (int c = 0; c < CPU_SETSIZE; ++c)
            if (CPU_ISSET(c, &now)) s.allowed.push_back(c);
    s.owned.insert(detail::self_tid());
    return true;
}

/// The calling thread belongs to the pool or is the host: the sweep leaves it alone.
inline void note_owned_thread() {
    detail::State& s = detail::state();
    std::lock_guard<std::mutex> lk(s.mu);
    s.owned.insert(detail::self_tid());
}

/// Moves every thread of the process that is not owned to the aux set (or, with the feature off, does nothing).
/// Returns the number of threads whose placement changed.
inline int sweep() {
    detail::State& s = detail::state();
    if (!s.enabled.load(std::memory_order_acquire)) return 0;
    std::lock_guard<std::mutex> lk(s.mu);
    detail::purge_locked(s);
    int n = 0;
    for (int tid : detail::task_ids())
        if (s.owned.find(tid) == s.owned.end() && detail::move_locked(s, tid)) ++n;
    return n;
}

/// First thing a thread the engine creates does: it goes to the aux set while the feature is on.  Cheap when off.
inline void pin_current_thread() {
    detail::State& s = detail::state();
    if (!s.enabled.load(std::memory_order_acquire)) return;
    std::lock_guard<std::mutex> lk(s.mu);
    const int tid = detail::self_tid();
    if (s.owned.find(tid) == s.owned.end()) detail::move_locked(s, tid);
}

/// Puts back the placement every moved thread had.  Returns how many were restored.
inline int restore() {
    detail::State& s = detail::state();
    std::lock_guard<std::mutex> lk(s.mu);
    int n = 0;
    for (const auto& [tid, mask] : s.original)
        if (detail::alive(tid) && sched_setaffinity(tid, sizeof mask, &mask) == 0) ++n;
    s.original.clear();
    return n;
}

/// Turns the placement on (sweeping at once) or off (restoring the moved threads).  A no-op without a plan.
inline void set_enabled(bool on) {
    detail::State& s = detail::state();
    if (on) {
        if (!s.have_plan.load(std::memory_order_acquire)) return;
        s.enabled.store(true, std::memory_order_release);
        (void) sweep();
    } else {
        s.enabled.store(false, std::memory_order_release);
        (void) restore();
    }
}

inline bool enabled() { return detail::state().enabled.load(std::memory_order_acquire); }

namespace detail {
/// The CPUs a thread that must not share the host's core runs on: every CPU the process may use but the host's and
/// the pool workers' (on an SMT machine with the default pool, the idle sibling threads); without such CPUs, every CPU
/// but the host's; failing that, every CPU.  Caller holds `mu`.  False when `configure` saw no CPUs.
inline bool off_host_target_locked(const State& s, cpu_set_t& out) {
    if (s.allowed.empty()) return false;
    auto build = [&](bool skip_workers) {
        CPU_ZERO(&out);
        int n = 0;
        for (int c : s.allowed) {
            if (c == s.host_cpu) continue;
            if (skip_workers && std::find(s.workers.begin(), s.workers.end(), c) != s.workers.end()) continue;
            CPU_SET(c, &out);
            ++n;
        }
        return n;
    };
    if (build(true) > 0 || build(false) > 0) return true;
    CPU_ZERO(&out);
    for (int c : s.allowed) CPU_SET(c, &out);
    return true;
}
}  // namespace detail

/// Puts the calling thread off the host's core and off the pool workers' CPUs (see off_host_target_locked).  A thread
/// the host creates inherits the host's single CPU; one that must not share that core (the concurrent prompt read's
/// thread and its doorbell thread, STRATA_PIPED_CONCURRENT) calls this first.  False when nothing changed.
inline bool place_off_host() {
    detail::State& s = detail::state();
    std::lock_guard<std::mutex> lk(s.mu);
    cpu_set_t to;
    if (!detail::off_host_target_locked(s, to)) return false;
    return sched_setaffinity(0, sizeof to, &to) == 0;
}

/// Called on the host thread: every other thread that runs on exactly the host's CPUs (it inherited them: the prefill
/// copy threads started by the first prompt, a helper the host started) goes where place_off_host puts a thread.  The
/// pool's workers and the host threads are never moved.  Returns how many moved.
inline int release_host_inheritors() {
    detail::State& s = detail::state();
    std::lock_guard<std::mutex> lk(s.mu);
    cpu_set_t host, all, to;
    CPU_ZERO(&host);
    CPU_ZERO(&all);
    if (!detail::off_host_target_locked(s, to) || sched_getaffinity(0, sizeof host, &host) != 0) return 0;
    for (int c : s.allowed) CPU_SET(c, &all);
    if (detail::same_mask(host, all)) return 0;   // the host is not pinned: nothing inherited a single core
    detail::purge_locked(s);
    const int self = detail::self_tid();
    int n = 0;
    for (int tid : detail::task_ids()) {
        if (tid == self || s.owned.find(tid) != s.owned.end()) continue;
        cpu_set_t m;
        CPU_ZERO(&m);
        if (sched_getaffinity(tid, sizeof m, &m) != 0 || !detail::same_mask(m, host)) continue;
        if (sched_setaffinity(tid, sizeof to, &to) == 0) ++n;
    }
    return n;
}

/// The aux CPUs of the plan ({} when there is none).
inline std::vector<int> plan() {
    detail::State& s = detail::state();
    std::lock_guard<std::mutex> lk(s.mu);
    return s.plan;
}

/// Reads sysfs for the cores of the process's CPUs.  `cpus` empty: every CPU `configure` saw.
inline std::vector<LogicalCpu> read_topology(const std::vector<int>& cpus) {
    auto read_id = [](int cpu, const char* what) -> long {
        char path[96];
        std::snprintf(path, sizeof path, "/sys/devices/system/cpu/cpu%d/topology/%s", cpu, what);
        long v = -1;
        if (std::FILE* f = std::fopen(path, "r")) {
            if (std::fscanf(f, "%ld", &v) != 1) v = -1;
            std::fclose(f);
        }
        return v;
    };
    std::vector<LogicalCpu> out;
    for (int c : cpus) out.push_back({c, read_id(c, "physical_package_id"), read_id(c, "core_id")});
    return out;
}

/// Works the plan out once the pool exists: `host_cpu` is the CPU the host thread is pinned to (-1: unknown) and
/// `worker_cpus` the CPUs of the pool's workers.  With the start-up setting on, the placement starts now.  Returns one
/// line for the log.
inline std::string set_topology(int host_cpu, const std::vector<int>& worker_cpus) {
    detail::State& s = detail::state();
    std::vector<int> plan;
    std::string note;
    {
        std::lock_guard<std::mutex> lk(s.mu);
        const std::vector<LogicalCpu> allowed = read_topology(s.allowed);
        plan = s.cfg.use_list ? restrict_list(s.cfg.list, allowed, host_cpu, worker_cpus)
                              : derive(allowed, host_cpu, worker_cpus);
        s.plan = plan;
        s.workers = worker_cpus;
        s.host_cpu = host_cpu;
        CPU_ZERO(&s.target);
        for (int c : plan) CPU_SET(c, &s.target);
        s.have_plan.store(!plan.empty(), std::memory_order_release);
        for (size_t i = 0; i < plan.size(); ++i) note += (i ? "," : "") + std::to_string(plan[i]);
    }
    if (plan.empty()) note = "none spare (every CPU is the host's, a worker's or shares a core with a worker or has no known core)";
    if (s.cfg.on && !plan.empty()) set_enabled(true);
    long moved;
    {
        std::lock_guard<std::mutex> lk(s.mu);
        moved = (long) s.original.size();
    }
    return "strata aux cpus: threads that are neither pool workers nor the host go to CPUs " + note +
           (!s.cfg.on ? " (off at start; a request's aux_cpus=1 turns it on)"
            : plan.empty() ? " (nothing to do)" : " (on; " + std::to_string(moved) + " threads placed so far)");
}

#else  // not Linux: the same names, doing nothing

inline bool configure(const std::string& spec, std::string& err) {
    Config cfg;
    return parse_spec(spec, cfg, err);
}
inline void note_owned_thread() {}
inline int sweep() { return 0; }
inline void pin_current_thread() {}
inline int restore() { return 0; }
inline void set_enabled(bool) {}
inline bool enabled() { return false; }
inline bool place_off_host() { return false; }   // (a new thread does not inherit the host's CPU set here)
inline int release_host_inheritors() { return 0; }
inline std::vector<int> plan() { return {}; }
inline std::string set_topology(int, const std::vector<int>&) { return "strata aux cpus: not available on this system"; }

#endif

}  // namespace strata::aux_cpus