Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
10.2 KB
Raw Blame
#pragma once

#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>

// The prompt path's chunk, streamed ring and expert-cache loan, priced against the HOST as well as the GPU (#2824).
//
// The automatic chunk scan only asks whether the loan fits the GPU cache.  But a lent slot's expert is not in VRAM
// during the prompt: it comes from the RAM copy when the copy kept it, else from the model file, and a chunk of
// stream_all_min_tokens or more streams EVERY expert the GPU does not hold, once per chunk.  When the file set
// (the experts neither the GPU cache nor the RAM copy hold, plus the displaced ones with no RAM backup) is bigger than
// the RAM the OS can keep file pages in, the set cycles through the page cache and each chunk reads most of it again:
// 2x RTX 3060 / 32 GB / IQ2_XS read 28-45 GB for one 29K prompt at 8192-token chunks (file set 15.1 GiB against a
// 14.5 GiB page cache) and 0.9-1.4 GB at 4096 (11.6 GiB).
//
// This is the pure model (no engine, no GPU) so it can be tested on the CPU.  The engine builds the Options (a chunk x
// ring grid, each with the bytes it lends and the bytes of those the RAM copy keeps) and `choose` returns the one with
// the least predicted prompt time; a FlexGen-style joint choice of placement (what is lent) and I/O schedule (how many
// passes over the file set): when the set can never fit, f is ~1 for every option and the fewest passes (the biggest
// chunk) wins, so nothing is "rejected" and nothing is shrunk blindly.
//
// Time per prompt of N tokens in n = ceil(N / chunk) chunks:
//     n * (chunk-compute + stream(chunk) / h2d) + file_reads / ssd,   file_reads = n * f(W / page_cache) * W
// The streaming term is ADDITIVE with the compute term: the 12 GB-card 32K Q2_0 prompt measured 791 / 878 / 973 tok/s at
// 4096 / 6144 / 8192, which is 1,275 tok/s of compute plus ~1.9 s of streaming per chunk (a 20 GB stream at ~10 GB/s).
namespace strata::program::prefill_plan {

constexpr double kGiB = 1073741824.0;

struct Host {
    uint64_t page_cache = 0;     ///< bytes the OS can keep file pages in while a prompt runs (0 = reads are unbuffered)
    double h2d_gbps = 12.0;      ///< host -> device, GB/s (the engine's probe)
    double ssd_gbps = 0.45;      ///< file tier reads, GB/s (rotational ~0.15, SATA ~0.45, NVMe ~2.5)
    double compute_tok_s = 1200; ///< prompt compute without streaming, tokens/s (only scales the printed seconds)
    double ring_loss = 0.05;     ///< prompt time lost between a full ring and the smallest one (#583: 17 vs 199 slots)
};

struct Tier {
    uint64_t experts_total = 0;  ///< every routed expert of the pack
    uint64_t gpu_cache = 0;      ///< bytes of all GPU expert caches (lent slots included)
    uint64_t file_tier = 0;      ///< experts neither a GPU cache nor the RAM copy holds (the decode-time file set)
    uint64_t extra_file = 0;     ///< other file pages a prompt touches (PLE rows)
};

struct Option {
    int64_t chunk = 0;
    int64_t ring = 0;            ///< the ring slots the prompt path will really lay out
    int64_t ring_full = 0;       ///< the largest ring on offer (0 = unknown / not adjustable)
    uint64_t lend_bytes = 0;     ///< cache bytes the loan displaces
    uint64_t lend_kept = 0;      ///< ...of which the RAM copy keeps a copy
};

struct Cost {
    Option opt;
    double file_set = 0;         ///< bytes the prompt path reads through the file tier, one chunk
    double miss = 0;             ///< share of that set each chunk fetches from the drive
    double read_bytes = 0;       ///< drive bytes per prompt (steady state: the set already warm)
    double seconds = 0;          ///< predicted prompt time
    int64_t chunks = 0;
    bool fits = false;           ///< the set sits in the page cache with room to spare
};

/// The share of a chunk's file set that comes from the drive, from r = file_set / page_cache, with page_cache = the RAM
/// available after the copy less a slack.  Fitted to two rigs that disagree below r = 1 and agree above it:
///   #2824 (2x RTX 3060, 32 GB, SATA, 4 chunk sizes): r 0.80 read ~1 GB per prompt, r 0.92 1.2-2.8 GB, r 1.04 a miss share of 0.47-0.75;
///   P2003 (RTX 3060, IQ2_XS, cgroup 32G, 5 chunk sizes, 29K prompt): r 0.96 0.38, 1.00 0.44, 1.03 0.49, 1.07 0.61, 1.12 0.74.
/// Below 0.85 the set stays cached; from there the share climbs (the page cache is never exactly the size it looks: the
/// PLE shard and the startup reads keep pages in it), and past ~1.3 nearly every page misses.
inline double miss_share(double r) {
    if (r <= 0.85) return 0.0;
    if (r <= 1.00) return 0.40 * (r - 0.85) / 0.15;
    if (r <= 1.12) return 0.40 + 0.35 * (r - 1.00) / 0.12;
    if (r <= 1.30) return 0.75 + 0.25 * (r - 1.12) / 0.18;
    return 1.0;
}

inline Cost price(const Host& h, const Tier& t, int64_t tokens, const Option& o) {
    Cost c;
    c.opt = o;
    c.chunks = o.chunk > 0 ? (tokens + o.chunk - 1) / o.chunk : 1;
    const uint64_t unbacked = o.lend_bytes > o.lend_kept ? o.lend_bytes - o.lend_kept : 0;
    c.file_set = (double) t.file_tier + (double) unbacked + (double) t.extra_file;
    const double r = h.page_cache > 0 ? c.file_set / (double) h.page_cache : 1e9;   // unbuffered: nothing is cached
    c.miss = h.page_cache > 0 ? miss_share(r) : 1.0;
    c.fits = h.page_cache > 0 && r <= 0.85;
    c.read_bytes = (double) c.chunks * c.file_set * c.miss;
    // a chunk streams every expert the GPU does not hold: the loan returns to that set for the length of the prompt
    const double stream = std::max(0.0, (double) t.experts_total - (double) t.gpu_cache + (double) o.lend_bytes);
    double ring_pen = 1.0;
    if (o.ring_full > 16 && o.ring < o.ring_full)
        ring_pen = 1.0 + h.ring_loss * (1.0 - (double) (o.ring - 16) / (double) (o.ring_full - 16));
    const double compute = (double) tokens / h.compute_tok_s * ring_pen;
    c.seconds = compute + (double) c.chunks * stream / (h.h2d_gbps * 1e9) + c.read_bytes / (h.ssd_gbps * 1e9);
    return c;
}

struct Pick {
    Cost best, baseline;
    bool changed = false;
};

/// The option with the least predicted time.  `baseline` (today's choice) is kept unless the pick saves at least
/// `min_gain` of its time AND at least `min_save_s` seconds: a plan that fits the page cache is never touched, and
/// near-ties keep the established behaviour.
inline Pick choose(const Host& h, const Tier& t, int64_t tokens, const std::vector<Option>& options, size_t baseline,
                   double min_gain = 0.05, double min_save_s = 2.0) {
    Pick p;
    p.baseline = price(h, t, tokens, options.at(baseline));
    p.best = p.baseline;
    for (size_t i = 0; i < options.size(); ++i) {
        if (i == baseline) continue;
        const Cost c = price(h, t, tokens, options[i]);
        if (c.seconds < p.best.seconds - 1e-9 ||
            (std::abs(c.seconds - p.best.seconds) <= 1e-9 && c.opt.chunk > p.best.opt.chunk))
            p.best = c;
    }
    p.changed = p.best.opt.chunk != p.baseline.opt.chunk || p.best.opt.ring != p.baseline.opt.ring;
    if (p.changed && !(p.baseline.seconds - p.best.seconds >= min_gain * p.baseline.seconds &&
                       p.baseline.seconds - p.best.seconds >= min_save_s)) {
        p.best = p.baseline;
        p.changed = false;
    }
    return p;
}

/// The chunk a request of `tokens` reads in when the plan chose `plan` < `cap` (the buffers' chunk, what the VRAM scan picked): a
/// prompt that fits one chunk of `cap` keeps ONE chunk (rounded to 256) - one pass reads each expert once, so it cannot thrash the
/// page cache and splitting it only adds a pass - and a longer one reads in n = ceil(tokens / plan) chunks of equal size (rounded up
/// to 256, at most `cap`), so there is never a stub of a tail.
inline int64_t balanced_chunk(int64_t tokens, int64_t cap, int64_t plan) {
    if (tokens <= 0 || cap <= 0) return 0;
    const int64_t one = (tokens + 255) / 256 * 256;
    if (plan <= 0 || plan >= cap || one <= cap) return std::min(one, cap);
    // n chunks of the plan's size or a little less (never more: the loan the plan priced stays the loan), the first size whose last
    // chunk is at least 40% of a chunk; failing that, the one with the longest last chunk
    const int64_t n = (tokens + plan - 1) / plan;
    int64_t c = std::min((tokens + n - 1) / n + 255, plan + 255) / 256 * 256, best = c;
    double best_tail = -1;
    for (int k = 0; k < 8 && c >= 256; ++k, c -= 256) {
        const int64_t nn = (tokens + c - 1) / c, tail = tokens - (nn - 1) * c;
        if (tail * 5 >= c * 2) return c;
        if ((double) tail / (double) c > best_tail) { best_tail = (double) tail / (double) c; best = c; }
    }
    return best;
}

/// One log line: what was chosen, why, and the reads it expects per prompt.
inline std::string describe(const Pick& p, const Host& h, int64_t tokens) {
    char b[768];
    const Cost& c = p.best;
    if (!p.changed) {
        std::snprintf(b, sizeof b,
                      "strata: prefill plan: chunk %lld, ring %lld kept: file set %.1f GiB %s page cache %.1f GiB -> about %.1f GB "
                      "read from the drive per %lld-token prompt (%lld chunks)%s",
                      (long long) c.opt.chunk, (long long) c.opt.ring, c.file_set / kGiB,
                      c.fits ? "fits the" : "does not fit the", (double) h.page_cache / kGiB, c.read_bytes / 1e9,
                      (long long) tokens, (long long) c.chunks,
                      c.fits ? "" : "; no option does better (the set cannot fit: fewest passes win)");
    } else {
        const Cost& o = p.baseline;
        std::snprintf(b, sizeof b,
                      "strata: prefill plan: chunk %lld -> %lld, ring %lld -> %lld: today's choice has a %.1f GiB file set "
                      "against a %.1f GiB page cache (about %.1f GB read from the drive per %lld-token prompt, %.0f s); this one "
                      "%.1f GiB, about %.1f GB read, %.0f s. --prefill N keeps your own chunk",
                      (long long) o.opt.chunk, (long long) c.opt.chunk, (long long) o.opt.ring, (long long) c.opt.ring,
                      o.file_set / kGiB, (double) h.page_cache / kGiB, o.read_bytes / 1e9, (long long) tokens, o.seconds,
                      c.file_set / kGiB, c.read_bytes / 1e9, c.seconds);
    }
    return b;
}

}  // namespace strata::program::prefill_plan