Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Prefill planner: price the host (RAM page cache, displaced experts, drive) when choosing chunk and ring (#2824)
Niko1221e77f1dfc3fa1
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 9591fe3f8..3a265785d 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -293,6 +293,11 @@ if(STRATA_BUILD_TESTS)target_include_directories(split_rules_test PRIVATE include)add_test(NAME split_rules_test COMMAND split_rules_test)+ # the prompt planner's host-memory model (#2824): header-only, no GPU+ add_executable(prefill_plan_test tests/program/prefill_plan_test.cpp)+ target_include_directories(prefill_plan_test PRIVATE include)+ add_test(NAME prefill_plan_test COMMAND prefill_plan_test)+# the default CPU prefill share is not armed when every expert is resident (#1595): header-only rule, no GPUadd_executable(share_rules_test tests/prefill/share_rules_test.cpp)target_include_directories(share_rules_test PRIVATE include)diff --git a/include/strata/program/prefill_plan.hpp b/include/strata/program/prefill_plan.hppnew file mode 100644index 000000000..60d717416--- /dev/null+++ b/include/strata/program/prefill_plan.hpp@@ -0,0 +1,152 @@+#pragma once++#include <algorithm>+#include <cmath>+#include <cstdint>+#include <cstdio>+#include <string>+#include <vector>++// The prompt path's chunk, streamed ring and expert-cache loan, priced against the HOST as well as the GPU (#2824).+//+// The automatic chunk scan only asks whether the loan fits the GPU cache. But a lent slot's expert is not in VRAM+// during the prompt: it comes from the RAM copy when the copy kept it, else from the model file, and a chunk of+// stream_all_min_tokens or more streams EVERY expert the GPU does not hold, once per chunk. When the file set+// (the experts neither the GPU cache nor the RAM copy hold, plus the displaced ones with no RAM backup) is bigger than+// the RAM the OS can keep file pages in, the set cycles through the page cache and each chunk reads most of it again:+// 2x RTX 3060 / 32 GB / IQ2_XS read 28-45 GB for one 29K prompt at 8192-token chunks (file set 15.1 GiB against a+// 14.5 GiB page cache) and 0.9-1.4 GB at 4096 (11.6 GiB).+//+// This is the pure model (no engine, no GPU) so it can be tested on the CPU. The engine builds the Options (a chunk x+// ring grid, each with the bytes it lends and the bytes of those the RAM copy keeps) and `choose` returns the one with+// the least predicted prompt time; a FlexGen-style joint choice of placement (what is lent) and I/O schedule (how many+// passes over the file set): when the set can never fit, f is ~1 for every option and the fewest passes (the biggest+// chunk) wins, so nothing is "rejected" and nothing is shrunk blindly.+//+// Time per prompt of N tokens in n = ceil(N / chunk) chunks:+// n * (chunk-compute + stream(chunk) / h2d) + file_reads / ssd, file_reads = n * f(W / page_cache) * W+// The streaming term is ADDITIVE with the compute term: the 12 GB-card 32K Q2_0 prompt measured 791 / 878 / 973 tok/s at+// 4096 / 6144 / 8192, which is 1,275 tok/s of compute plus ~1.9 s of streaming per chunk (a 20 GB stream at ~10 GB/s).+namespace strata::program::prefill_plan {++constexpr double kGiB = 1073741824.0;++struct Host {+ uint64_t page_cache = 0; ///< bytes the OS can keep file pages in while a prompt runs (0 = reads are unbuffered)+ double h2d_gbps = 12.0; ///< host -> device, GB/s (the engine's probe)+ double ssd_gbps = 0.45; ///< file tier reads, GB/s (rotational ~0.15, SATA ~0.45, NVMe ~2.5)+ double compute_tok_s = 1200; ///< prompt compute without streaming, tokens/s (only scales the printed seconds)+ double ring_loss = 0.05; ///< prompt time lost between a full ring and the smallest one (#583: 17 vs 199 slots)+};++struct Tier {+ uint64_t experts_total = 0; ///< every routed expert of the pack+ uint64_t gpu_cache = 0; ///< bytes of all GPU expert caches (lent slots included)+ uint64_t file_tier = 0; ///< experts neither a GPU cache nor the RAM copy holds (the decode-time file set)+ uint64_t extra_file = 0; ///< other file pages a prompt touches (PLE rows)+};++struct Option {+ int64_t chunk = 0;+ int64_t ring = 0; ///< the ring slots the prompt path will really lay out+ int64_t ring_full = 0; ///< the largest ring on offer (0 = unknown / not adjustable)+ uint64_t lend_bytes = 0; ///< cache bytes the loan displaces+ uint64_t lend_kept = 0; ///< ...of which the RAM copy keeps a copy+};++struct Cost {+ Option opt;+ double file_set = 0; ///< bytes the prompt path reads through the file tier, one chunk+ double miss = 0; ///< share of that set each chunk fetches from the drive+ double read_bytes = 0; ///< drive bytes per prompt (steady state: the set already warm)+ double seconds = 0; ///< predicted prompt time+ int64_t chunks = 0;+ bool fits = false; ///< the set sits in the page cache with room to spare+};++/// The share of a chunk's file set that comes from the drive, from r = file_set / page_cache. Fitted to #2824:+/// r 0.80 (4096 chunks) read ~1 GB per prompt, r 0.92 (6144) 1.2-2.8 GB, r 1.04 (8192) 28-45 GB over four chunks+/// (a miss share of 0.47-0.75). Below 0.80 the set stays cached; the ramp to 1.0 is the page cache's imperfect LRU on+/// a cyclic scan (some pages survive); well past 1 nearly everything misses.+inline double miss_share(double r) {+ if (r <= 0.80) return 0.0;+ if (r <= 1.00) return 0.15 * (r - 0.80) / 0.20;+ if (r <= 1.05) return 0.15 + 0.50 * (r - 1.00) / 0.05;+ return std::min(1.0, 0.65 + 0.35 * (r - 1.05) / 0.25);+}++inline Cost price(const Host& h, const Tier& t, int64_t tokens, const Option& o) {+ Cost c;+ c.opt = o;+ c.chunks = o.chunk > 0 ? (tokens + o.chunk - 1) / o.chunk : 1;+ const uint64_t unbacked = o.lend_bytes > o.lend_kept ? o.lend_bytes - o.lend_kept : 0;+ c.file_set = (double) t.file_tier + (double) unbacked + (double) t.extra_file;+ const double r = h.page_cache > 0 ? c.file_set / (double) h.page_cache : 1e9; // unbuffered: nothing is cached+ c.miss = h.page_cache > 0 ? miss_share(r) : 1.0;+ c.fits = h.page_cache > 0 && r <= 0.80;+ c.read_bytes = (double) c.chunks * c.file_set * c.miss;+ // a chunk streams every expert the GPU does not hold: the loan returns to that set for the length of the prompt+ const double stream = std::max(0.0, (double) t.experts_total - (double) t.gpu_cache + (double) o.lend_bytes);+ double ring_pen = 1.0;+ if (o.ring_full > 16 && o.ring < o.ring_full)+ ring_pen = 1.0 + h.ring_loss * (1.0 - (double) (o.ring - 16) / (double) (o.ring_full - 16));+ const double compute = (double) tokens / h.compute_tok_s * ring_pen;+ c.seconds = compute + (double) c.chunks * stream / (h.h2d_gbps * 1e9) + c.read_bytes / (h.ssd_gbps * 1e9);+ return c;+}++struct Pick {+ Cost best, baseline;+ bool changed = false;+};++/// The option with the least predicted time. `baseline` (today's choice) is kept unless the pick saves at least+/// `min_gain` of its time AND at least `min_save_s` seconds: a plan that fits the page cache is never touched, and+/// near-ties keep the established behaviour.+inline Pick choose(const Host& h, const Tier& t, int64_t tokens, const std::vector<Option>& options, size_t baseline,+ double min_gain = 0.05, double min_save_s = 2.0) {+ Pick p;+ p.baseline = price(h, t, tokens, options.at(baseline));+ p.best = p.baseline;+ for (size_t i = 0; i < options.size(); ++i) {+ if (i == baseline) continue;+ const Cost c = price(h, t, tokens, options[i]);+ if (c.seconds < p.best.seconds - 1e-9 ||+ (std::abs(c.seconds - p.best.seconds) <= 1e-9 && c.opt.chunk > p.best.opt.chunk))+ p.best = c;+ }+ p.changed = p.best.opt.chunk != p.baseline.opt.chunk || p.best.opt.ring != p.baseline.opt.ring;+ if (p.changed && !(p.baseline.seconds - p.best.seconds >= min_gain * p.baseline.seconds &&+ p.baseline.seconds - p.best.seconds >= min_save_s)) {+ p.best = p.baseline;+ p.changed = false;+ }+ return p;+}++/// One log line: what was chosen, why, and the reads it expects per prompt.+inline std::string describe(const Pick& p, const Host& h, int64_t tokens) {+ char b[768];+ const Cost& c = p.best;+ if (!p.changed) {+ std::snprintf(b, sizeof b,+ "strata: prefill plan: chunk %lld, ring %lld kept: file set %.1f GiB %s page cache %.1f GiB -> about %.1f GB "+ "read from the drive per %lld-token prompt (%lld chunks)%s",+ (long long) c.opt.chunk, (long long) c.opt.ring, c.file_set / kGiB,+ c.fits ? "fits the" : "does not fit the", (double) h.page_cache / kGiB, c.read_bytes / 1e9,+ (long long) tokens, (long long) c.chunks,+ c.fits ? "" : "; no option does better (the set cannot fit: fewest passes win)");+ } else {+ const Cost& o = p.baseline;+ std::snprintf(b, sizeof b,+ "strata: prefill plan: chunk %lld -> %lld, ring %lld -> %lld: today's choice has a %.1f GiB file set "+ "against a %.1f GiB page cache (about %.1f GB read from the drive per %lld-token prompt, %.0f s); this one "+ "%.1f GiB, about %.1f GB read, %.0f s. --prefill N keeps your own chunk",+ (long long) o.opt.chunk, (long long) c.opt.chunk, (long long) o.opt.ring, (long long) c.opt.ring,+ o.file_set / kGiB, (double) h.page_cache / kGiB, o.read_bytes / 1e9, (long long) tokens, o.seconds,+ c.file_set / kGiB, c.read_bytes / 1e9, c.seconds);+ }+ return b;+}++} // namespace strata::program::prefill_plandiff --git a/src/program/generate.cpp b/src/program/generate.cppindex 3035bf166..70ee83a0b 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -76,6 +76,7 @@#include "strata/prefill/share_rules.hpp"#include "strata/platform/host_limits.hpp"#include "strata/program/split_ring.hpp"+#include "strata/program/prefill_plan.hpp"#include "strata/program/draft_kv_plan.hpp"#include "strata/program/message_boundary.hpp"#include "strata/spec/draft_policy.hpp"@@ -1349,6 +1350,24 @@ void disk_wait_threads(std::FILE* f) {}#endif+// #2824: what the drive under `path` reads at, GB/s, for the prompt planner (it prices the file set a chunk re-reads). Linux:+// rotational disk 0.15, NVMe 2.5, other 0.45 (SATA); elsewhere 1.5. STRATA_SSD_GBPS overrides. `why` says where it came from.+double drive_read_gbps(const std::string& path, std::string& why) {+ if (const char* e = std::getenv("STRATA_SSD_GBPS"); e != nullptr && std::atof(e) > 0.0) {+ why = "STRATA_SSD_GBPS";+ return std::atof(e);+ }+#if !defined(_WIN32)+ std::string dev;+ const int rot = on_rotational_disk(path, dev);+ if (rot == 1) { why = dev + " is a rotational disk"; return 0.15; }+ if (rot == 0 && dev.rfind("nvme", 0) == 0) { why = dev + " is NVMe"; return 2.5; }+ if (rot == 0) { why = dev + " is a SATA-class SSD (assumed)"; return 0.45; }+#endif+ why = "assumed";+ return 1.5;+}+// --pipeline-windows: what the watchdog prints about the pipelined decode loop (set while it runs)struct PlDiag {strata::core::Verifier* v[2][2] = {};@@ -1951,6 +1970,7 @@ static bool pcie_auto_split_ok() {// median would still follow a disturbance that lasts through half the bursts). Each burst takes ~10 ms on an x16// PCIe 4 link, so the probe takes no longer than the one 1 GiB burst did. `samples`, when given, gets every// burst's reading for the log.+double g_last_h2d_gbps = -1.0; // the last probe's reading, for the prompt planner (#2824)double probe_pcie_h2d_gbps(std::string* samples = nullptr) {constexpr size_t kBytes = 256ull << 20;constexpr int kBursts = 4;@@ -2006,6 +2026,7 @@ double probe_pcie_h2d_gbps(std::string* samples = nullptr) {if (!ok) (void) cudaGetLastError();cudaFree(d);cudaFreeHost(h);+ if (bw > 0.0) g_last_h2d_gbps = bw;return bw;}@@ -7380,6 +7401,109 @@ int main(int argc, char** argv) {}return 0;};+ // ---- #2824: THE PROMPT PLAN PRICES THE HOST. The scans above ask whether a loan fits the GPU cache. But a lent+ // slot's expert is not in VRAM for the length of the prompt: it comes from the RAM copy if the copy kept it, else from+ // the model file, and a chunk streams every expert the GPU does not hold once per chunk. When that file set is bigger+ // than the RAM the OS keeps file pages in, it cycles through the page cache and each chunk reads most of it again+ // (2x RTX 3060, 32 GB, IQ2_XS, 29K prompt: 8192-token chunks read 28-45 GB, 4096 read ~1 GB). So the auto pick is+ // refined by prefill_plan (include/strata/program/prefill_plan.hpp): the chunk x ring grid at or below the VRAM pick,+ // each priced with the expert bytes its loan displaces without a RAM backup, the RAM left for the page cache, the+ // measured PCIe rate and a drive-speed class, and the least predicted prompt time wins. A set that fits the page cache+ // keeps today's pick untouched. Advisory: --prefill N and STRATA_PREFILL_RING are the operator's, and+ // STRATA_PREFILL_HOST_PLAN=0 turns this off. No weights or experts change; only the chunk shape (rounding).+ // fits_vram(c) whether every participant can lend chunk c at the ring currently set+ // lend_of(c, bytes, slots) the cache bytes and slots the loan of chunk c takes at the ring currently set+ // ring_full(c) the largest ring slot count on offer for chunk c+ // restore() puts the ring state back as the VRAM pick left it+ // Returns the chunk to use (== `chunk` when nothing changes); the ring state is set for it.+ auto host_plan_chunk = [&](int64_t chunk, uint64_t gpu_cache_bytes, int64_t cache_slots,+ const std::function<bool(int64_t)>& fits_vram,+ const std::function<void(int64_t, uint64_t&, int64_t&)>& lend_of,+ const std::function<int64_t(int64_t)>& ring_full,+ const std::function<void()>& restore, int64_t tokens_hint = 0) -> int64_t {+ namespace pp = strata::program::prefill_plan;+ const char* off = std::getenv("STRATA_PREFILL_HOST_PLAN");+ if (!o.prefill_auto || chunk <= 0 || srcp != &src || (off != nullptr && off[0] == '0') ||+ !strata::prefill::Prefill::ring_bytes_enabled())+ return chunk;+ const auto& lay = strata::kernels::cpu::expert_layout();+ uint64_t total = 0;+ for (int64_t l = 0; l < g.n_layers; ++l) total += (uint64_t) lay.blob_bytes(l) * (uint64_t) g.n_expert;+ if (total == 0 || gpu_cache_bytes >= total) return chunk; // the GPU holds every expert: no file tier+ const auto hm = strata::core::sample_host_memory();+ if (!hm) return chunk;+ const uint64_t slack = 1200ull << 20; // MemAvailable overstates what the cache can hold (~1 GiB on the #2824 box)+ pp::Host host;+ host.page_cache = src.unbuffered() ? 0 : (hm->available > slack ? hm->available - slack : 0);+ host.h2d_gbps = g_last_h2d_gbps > 0.0 ? g_last_h2d_gbps : 12.0;+ std::string why;+ host.ssd_gbps = drive_read_gbps(o.pack, why);+ if (const char* e = std::getenv("STRATA_PREFILL_PLAN_TOKS"); e != nullptr && std::atof(e) > 0.0)+ host.compute_tok_s = std::atof(e);+ const uint64_t ram_copy = src.complement_ready() ? src.resident_bytes() : 0;+ const int64_t kept_slots = src.complement_ready() ? src.resident_lent_slots() : 0;+ const uint64_t slot_avg = cache_slots > 0 ? gpu_cache_bytes / (uint64_t) cache_slots : 0;+ const uint64_t kept_bytes = (uint64_t) std::max<int64_t>(0, kept_slots) * slot_avg; // the copy's lend region+ pp::Tier tier;+ tier.experts_total = total;+ tier.gpu_cache = gpu_cache_bytes;+ const uint64_t ram_outside_gpu = ram_copy > kept_bytes ? ram_copy - kept_bytes : 0;+ tier.file_tier = total > gpu_cache_bytes + ram_outside_gpu ? total - gpu_cache_bytes - ram_outside_gpu : 0;+ tier.extra_file = o.ple_gguf.empty() ? 0 : (350ull << 20); // PLE rows the prompt touches (#2824: ~0.35 GiB)+ const int64_t tokens = tokens_hint > 0 ? tokens_hint+ : std::max<int64_t>(chunk, std::min<int64_t>(std::max<int64_t>(o.max_context, 4096), 32768));+ std::vector<pp::Option> opts;+ auto add = [&](int64_t c, int64_t ring_req) {+ strata::prefill::Prefill::set_ring_budget((int) ring_req, 0);+ if (!fits_vram(c)) return;+ pp::Option op;+ op.chunk = c;+ op.ring = strata::prefill::Prefill::ring_slots_for(c);+ op.ring_full = ring_full(c);+ uint64_t lb = 0;+ int64_t ls = 0;+ lend_of(c, lb, ls);+ op.lend_bytes = lb;+ op.lend_kept = std::min<uint64_t>(lb, kept_bytes);+ for (const pp::Option& e : opts) if (e.chunk == op.chunk && e.ring == op.ring) return;+ opts.push_back(op);+ };+ // today's pick first, priced under the ring state it left+ restore();+ pp::Option base;+ base.chunk = chunk;+ base.ring = strata::prefill::Prefill::ring_slots_for(chunk);+ base.ring_full = ring_full(chunk);+ {+ uint64_t lb = 0;+ int64_t ls = 0;+ lend_of(chunk, lb, ls);+ base.lend_bytes = lb;+ base.lend_kept = std::min<uint64_t>(lb, kept_bytes);+ }+ opts.push_back(base);+ for (int64_t c = chunk; c >= 512; c -= 256) {+ const int64_t full = ring_full(c);+ add(c, (int) full);+ if (full > 32) add(c, (int) (full / 2));+ if (full > 16) add(c, 16);+ }+ pp::Pick pick = pp::choose(host, tier, tokens, opts, 0);+ std::fprintf(stderr, "%s\n", pp::describe(pick, host, tokens).c_str());+ std::fprintf(stderr, "strata: prefill plan inputs: %.1f GiB experts, %.1f GiB in the GPU caches, %.1f GiB RAM copy "+ "(%lld lend slots kept), file tier %.1f GiB; %.1f GiB available for the page cache%s; PCIe %.1f GB/s; "+ "drive %.2f GB/s (%s); a %lld-token prompt\n",+ (double) total / pp::kGiB, (double) gpu_cache_bytes / pp::kGiB, (double) ram_copy / pp::kGiB,+ (long long) kept_slots, (double) tier.file_tier / pp::kGiB, (double) host.page_cache / pp::kGiB,+ src.unbuffered() ? " (the file tier reads unbuffered: none)" : "", host.h2d_gbps, host.ssd_gbps, why.c_str(),+ (long long) tokens);+ if (!pick.changed) {+ restore();+ return chunk;+ }+ strata::prefill::Prefill::set_ring_budget((int) pick.best.opt.ring, 0);+ return pick.best.opt.chunk;+ };// ---- THE ELASTIC K/V (--kv-grow; vmm.hpp). Allocated for the whole --max-context up front, the K/V of a 262K// context (3.35 GiB with the drafter's at int8) takes VRAM the expert cache could hold more experts in. Now the K/V pools hold physical memory only for the cells// the requests reach. When a request needs more, slots just below the prompt path's loan give up their experts@@ -8066,7 +8190,7 @@ int main(int argc, char** argv) {if (c == 0 && cap < strata::prefill::Prefill::ring_default_slots()) c = probe(kRingMin);return std::max(c, small); // never a smaller chunk than 0.1.39's};- const int64_t chunk = scan(nullptr);+ int64_t chunk = scan(nullptr);// #448: a small card in a layer split caps every stage's chunk (an RTX 3080's 512-slot cache held a// 32 GB card's split to 512 tokens: prompts 6.2x slower, decode the same). Named when it bites, so the// regression is one log line: each stage that cannot fund the chunk CUDA0 alone would read in.@@ -8081,7 +8205,37 @@ int main(int argc, char** argv) {small >= chunk ? 0 : (int) ring_room(chunk, nullptr, strata::prefill::Prefill::ring_cap_for(small)),small);}- if (chunk > 0 && alone > chunk) {+ // #2824: the VRAM pick, priced against the host (RAM left for the page cache, the displaced experts with no RAM+ // backup, the drive): a smaller chunk / ring when the file set the prompt re-reads outgrows the page cache.+ // The RAM copy is built by now, so its bytes and the lend slots it kept are known.+ const int64_t chunk_vram = chunk;+ if (chunk > 0 && o.prefill_auto && strata::prefill::Prefill::ring_bytes_enabled()) {+ const int64_t small0 = old_pick(nullptr);+ const int64_t cap0 = strata::prefill::Prefill::ring_cap_for(small0);+ const int64_t ring0 = small0 >= chunk ? 0 : ring_room(chunk, nullptr, cap0);+ auto restore_ring = [&] { strata::prefill::Prefill::set_ring_budget((int) ring0, small0); };+ restore_ring();+ uint64_t gc_bytes = 0;+ int64_t gc_slots = 0;+ for (const PfPart& p : pf_parts) {+ gc_bytes += (uint64_t) std::max<int64_t>(0, (int64_t) p.cache->bytes());+ gc_slots += p.cache->slots();+ }+ chunk = host_plan_chunk(+ chunk, gc_bytes, gc_slots, [&](int64_t c) { return fits(c, true, nullptr); },+ [&](int64_t c, uint64_t& bytes, int64_t& slots) {+ bytes = 0;+ slots = 0;+ for (const PfPart& p : pf_parts) {+ const int64_t k = part_slots(p, c);+ if (k <= 0) continue;+ slots += k;+ bytes += part_bytes(p, (int32_t) (p.cache->slots() - k));+ }+ },+ [&](int64_t c) { return ring_room(c, nullptr, cap0); }, restore_ring);+ }+ if (chunk > 0 && alone > chunk_vram) {for (size_t i = 1; i < pf_parts.size(); ++i) {const PfPart& p = pf_parts[i];if (fits_one(p, alone, o.prefill_auto)) continue;@@ -8096,7 +8250,7 @@ int main(int argc, char** argv) {std::fprintf(stderr, "strata serve: WARNING: prompt chunk %lld tokens, not %lld: CUDA%d (%s) ""has %lld expert-cache slots, and a %lld-token chunk borrows %lld of them ""(it must keep 128%s) - prompts read slower than on CUDA0 alone (#448)\n",- (long long) chunk, (long long) alone, dev, prop.name,+ (long long) chunk_vram, (long long) alone, dev, prop.name,(long long) p.cache->slots(), (long long) alone,(long long) part_slots(p, alone), pct.c_str());// the helper tiers start at CUDA1 without a split (and are enabled in order)@@ -14981,7 +15135,24 @@ int main(int argc, char** argv) {uint64_t borrow_bytes = 0;if (!o.no_prefill_borrow && !host_res.empty() && d_res != nullptr) {int64_t chunk = o.prefill_chunk;+ const int64_t chunk_in = chunk;int64_t k = plan_lend(chunk); // auto: the largest chunk that fits; fixed: halved to fit+ if (k > 0 && o.prefill_auto && n_batched > 1024) { // #2824: priced against the host, for THIS prompt's length+ const int64_t vram_chunk = chunk;+ chunk = host_plan_chunk(+ chunk, (uint64_t) std::max<int64_t>(0, (int64_t) xcache.bytes()), xcache.slots(),+ [&](int64_t c) {+ const int64_t kk = lend_slots(c);+ return kk + 128 <= xcache.slots() && kk * 100 <= kAutoLendPct * xcache.slots();+ },+ [&](int64_t c, uint64_t& bytes, int64_t& slots) {+ slots = lend_slots(c);+ bytes = bytes_from_slots(slots);+ },+ [&](int64_t) { return strata::prefill::Prefill::ring_cap_for(4096); },+ [&] { int64_t c2 = chunk_in; (void) plan_lend(c2); }, n_batched);+ if (chunk != vram_chunk) k = lend_slots(chunk);+ }const int64_t request_sized = equal_chunk(n_batched, chunk);if (k > 0 && request_sized < chunk) { // no bigger than this prompt segment needschunk = request_sized;diff --git a/tests/program/prefill_plan_test.cpp b/tests/program/prefill_plan_test.cppnew file mode 100644index 000000000..baa4c4de2--- /dev/null+++ b/tests/program/prefill_plan_test.cpp@@ -0,0 +1,96 @@+// The prompt planner's host-memory model (#2824): no GPU, no model.+#include <cstdio>+#include <cstdlib>+#include <vector>++#include "strata/program/prefill_plan.hpp"++using namespace strata::program::prefill_plan;++static int failed = 0;+#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); ++failed; } } while (0)++static uint64_t gib(double g) { return (uint64_t) (g * kGiB); }++// options at the lend sizes the reporter's engine printed: 2048 1,688 slots 2.3 GiB, 4096 2.97K 4.0, 6144 5.8, 8192 7.5+static std::vector<Option> issue_options(int64_t full_ring) {+ std::vector<Option> v;+ const struct { int64_t c; double g; } L[] = {{1024, 1.6}, {2048, 2.3}, {3072, 3.1}, {4096, 4.0}, {5120, 4.9}, {6144, 5.8}, {7168, 6.6}, {8192, 7.5}};+ for (const auto& l : L) v.push_back({l.c, full_ring, full_ring, gib(l.g), 0});+ return v;+}++int main() {+ // #2824: 2x RTX 3060, 32 GB, IQ2_XS: the experts total 33.0 GiB, the GPU caches 13.84, the RAM copy 12.0 of the rest+ Host h;+ h.page_cache = gib(14.5);+ h.h2d_gbps = 8.0;+ h.ssd_gbps = 0.45;+ Tier t;+ t.experts_total = gib(33.0);+ t.gpu_cache = gib(13.84);+ t.file_tier = gib(7.18);+ t.extra_file = gib(0.35);+ const auto opts = issue_options(64);+ // the file sets the reporter computed: 15.1 GiB at 8192, 13.3 at 6144, 11.6 at 4096+ CHECK(std::abs(price(h, t, 29000, opts[7]).file_set / kGiB - 15.03) < 0.15);+ CHECK(std::abs(price(h, t, 29000, opts[5]).file_set / kGiB - 13.33) < 0.15);+ CHECK(std::abs(price(h, t, 29000, opts[3]).file_set / kGiB - 11.53) < 0.15);+ // measured drive reads per 29K prompt: 8192: 28-45 GB, 4096: 0.9-1.4 GB, 6144: 1.2-2.8 GB+ const Cost c8 = price(h, t, 28700, opts[7]), c6 = price(h, t, 28700, opts[5]), c4 = price(h, t, 28700, opts[3]);+ std::printf("8192: set %.1f GiB miss %.2f reads %.1f GB %.0f s | 6144: %.1f GiB %.2f %.1f GB %.0f s | 4096: %.1f GiB %.2f %.1f GB %.0f s\n",+ c8.file_set / kGiB, c8.miss, c8.read_bytes / 1e9, c8.seconds, c6.file_set / kGiB, c6.miss, c6.read_bytes / 1e9,+ c6.seconds, c4.file_set / kGiB, c4.miss, c4.read_bytes / 1e9, c4.seconds);+ CHECK(c8.read_bytes / 1e9 > 28 && c8.read_bytes / 1e9 < 46);+ CHECK(c4.read_bytes == 0 && c4.fits);+ CHECK(c6.read_bytes / 1e9 < 8);+ // the planner leaves 8192 for the one that fits+ const Pick p = choose(h, t, 28700, opts, 7);+ std::printf("%s\n", describe(p, h, 28700).c_str());+ CHECK(p.changed);+ CHECK(p.best.opt.chunk >= 3072 && p.best.opt.chunk <= 4096);+ CHECK(p.best.read_bytes < 3e9);++ // a box where everything fits keeps today's choice+ Host big = h;+ big.page_cache = gib(40);+ const Pick q = choose(big, t, 28700, opts, 7);+ CHECK(!q.changed && q.best.opt.chunk == 8192 && q.best.read_bytes == 0);++ // a box whose file set can never fit (8 GB card + 32 GB RAM): no refusal, no blind shrink: the fewest passes win+ Tier hard;+ hard.experts_total = gib(48);+ hard.gpu_cache = gib(3.0);+ hard.file_tier = gib(40);+ hard.extra_file = gib(0.4);+ Host small = h;+ small.page_cache = gib(22);+ std::vector<Option> o2;+ for (int64_t c = 1024; c <= 8192; c += 1024) o2.push_back({c, 64, 64, gib(0.9 * (double) c / 1024.0), 0});+ const Pick r = choose(small, hard, 28700, o2, 7);+ std::printf("%s\n", describe(r, small, 28700).c_str());+ CHECK(r.best.opt.chunk == 8192);+ // and from a small today's choice it moves UP to fewer passes when nothing fits either way+ const Pick r2 = choose(small, hard, 28700, o2, 1);+ CHECK(r2.best.opt.chunk > 2048);++ // unbuffered reads (no page cache): every chunk reads its set; the biggest chunk reads least+ Host ub = h;+ ub.page_cache = 0;+ CHECK(price(ub, t, 28700, opts[7]).read_bytes < price(ub, t, 28700, opts[3]).read_bytes);++ // lent experts the RAM copy keeps leave the file set: 8192 with all 7.5 GiB kept is 7.5 GiB smaller+ Option kept = opts[7];+ kept.lend_kept = kept.lend_bytes;+ CHECK(price(h, t, 28700, kept).file_set < price(h, t, 28700, opts[7]).file_set - gib(7.4));+ CHECK(price(h, t, 28700, kept).fits);++ // a smaller ring only wins when it brings the set into the cache+ Tier edge = t;+ edge.file_tier = gib(6.2); // 8192 with the 64-slot ring: 6.2+7.5+.35 = 14.05 GiB, 0.97 of the cache+ std::vector<Option> o3 = {{8192, 64, 64, gib(7.5), 0}, {8192, 16, 64, gib(7.0), 0}};+ const Pick s = choose(h, edge, 28700, o3, 0);+ std::printf("%s\n", describe(s, h, 28700).c_str());+ std::printf("%s\n", failed ? "FAILED" : "prefill_plan_test: ok");+ return failed ? 1 : 0;+}