Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Prefill planner: price the host (RAM page cache, displaced experts, drive) when choosing chunk and ring (#2824)

Niko1221e77f1dfc3fa1
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9591fe3f8..3a265785d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -293,6 +293,11 @@ if(STRATA_BUILD_TESTS)
target_include_directories(split_rules_test PRIVATE include)
add_test(NAME split_rules_test COMMAND split_rules_test)
+ # the prompt planner's host-memory model (#2824): header-only, no GPU
+ add_executable(prefill_plan_test tests/program/prefill_plan_test.cpp)
+ target_include_directories(prefill_plan_test PRIVATE include)
+ add_test(NAME prefill_plan_test COMMAND prefill_plan_test)
+
# the default CPU prefill share is not armed when every expert is resident (#1595): header-only rule, no GPU
add_executable(share_rules_test tests/prefill/share_rules_test.cpp)
target_include_directories(share_rules_test PRIVATE include)
diff --git a/include/strata/program/prefill_plan.hpp b/include/strata/program/prefill_plan.hpp
new file mode 100644
index 000000000..60d717416
--- /dev/null
+++ b/include/strata/program/prefill_plan.hpp
@@ -0,0 +1,152 @@
+#pragma once
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <string>
+#include <vector>
+
+// The prompt path's chunk, streamed ring and expert-cache loan, priced against the HOST as well as the GPU (#2824).
+//
+// The automatic chunk scan only asks whether the loan fits the GPU cache. But a lent slot's expert is not in VRAM
+// during the prompt: it comes from the RAM copy when the copy kept it, else from the model file, and a chunk of
+// stream_all_min_tokens or more streams EVERY expert the GPU does not hold, once per chunk. When the file set
+// (the experts neither the GPU cache nor the RAM copy hold, plus the displaced ones with no RAM backup) is bigger than
+// the RAM the OS can keep file pages in, the set cycles through the page cache and each chunk reads most of it again:
+// 2x RTX 3060 / 32 GB / IQ2_XS read 28-45 GB for one 29K prompt at 8192-token chunks (file set 15.1 GiB against a
+// 14.5 GiB page cache) and 0.9-1.4 GB at 4096 (11.6 GiB).
+//
+// This is the pure model (no engine, no GPU) so it can be tested on the CPU. The engine builds the Options (a chunk x
+// ring grid, each with the bytes it lends and the bytes of those the RAM copy keeps) and `choose` returns the one with
+// the least predicted prompt time; a FlexGen-style joint choice of placement (what is lent) and I/O schedule (how many
+// passes over the file set): when the set can never fit, f is ~1 for every option and the fewest passes (the biggest
+// chunk) wins, so nothing is "rejected" and nothing is shrunk blindly.
+//
+// Time per prompt of N tokens in n = ceil(N / chunk) chunks:
+// n * (chunk-compute + stream(chunk) / h2d) + file_reads / ssd, file_reads = n * f(W / page_cache) * W
+// The streaming term is ADDITIVE with the compute term: the 12 GB-card 32K Q2_0 prompt measured 791 / 878 / 973 tok/s at
+// 4096 / 6144 / 8192, which is 1,275 tok/s of compute plus ~1.9 s of streaming per chunk (a 20 GB stream at ~10 GB/s).
+namespace strata::program::prefill_plan {
+
+constexpr double kGiB = 1073741824.0;
+
+struct Host {
+ uint64_t page_cache = 0; ///< bytes the OS can keep file pages in while a prompt runs (0 = reads are unbuffered)
+ double h2d_gbps = 12.0; ///< host -> device, GB/s (the engine's probe)
+ double ssd_gbps = 0.45; ///< file tier reads, GB/s (rotational ~0.15, SATA ~0.45, NVMe ~2.5)
+ double compute_tok_s = 1200; ///< prompt compute without streaming, tokens/s (only scales the printed seconds)
+ double ring_loss = 0.05; ///< prompt time lost between a full ring and the smallest one (#583: 17 vs 199 slots)
+};
+
+struct Tier {
+ uint64_t experts_total = 0; ///< every routed expert of the pack
+ uint64_t gpu_cache = 0; ///< bytes of all GPU expert caches (lent slots included)
+ uint64_t file_tier = 0; ///< experts neither a GPU cache nor the RAM copy holds (the decode-time file set)
+ uint64_t extra_file = 0; ///< other file pages a prompt touches (PLE rows)
+};
+
+struct Option {
+ int64_t chunk = 0;
+ int64_t ring = 0; ///< the ring slots the prompt path will really lay out
+ int64_t ring_full = 0; ///< the largest ring on offer (0 = unknown / not adjustable)
+ uint64_t lend_bytes = 0; ///< cache bytes the loan displaces
+ uint64_t lend_kept = 0; ///< ...of which the RAM copy keeps a copy
+};
+
+struct Cost {
+ Option opt;
+ double file_set = 0; ///< bytes the prompt path reads through the file tier, one chunk
+ double miss = 0; ///< share of that set each chunk fetches from the drive
+ double read_bytes = 0; ///< drive bytes per prompt (steady state: the set already warm)
+ double seconds = 0; ///< predicted prompt time
+ int64_t chunks = 0;
+ bool fits = false; ///< the set sits in the page cache with room to spare
+};
+
+/// The share of a chunk's file set that comes from the drive, from r = file_set / page_cache. Fitted to #2824:
+/// r 0.80 (4096 chunks) read ~1 GB per prompt, r 0.92 (6144) 1.2-2.8 GB, r 1.04 (8192) 28-45 GB over four chunks
+/// (a miss share of 0.47-0.75). Below 0.80 the set stays cached; the ramp to 1.0 is the page cache's imperfect LRU on
+/// a cyclic scan (some pages survive); well past 1 nearly everything misses.
+inline double miss_share(double r) {
+ if (r <= 0.80) return 0.0;
+ if (r <= 1.00) return 0.15 * (r - 0.80) / 0.20;
+ if (r <= 1.05) return 0.15 + 0.50 * (r - 1.00) / 0.05;
+ return std::min(1.0, 0.65 + 0.35 * (r - 1.05) / 0.25);
+}
+
+inline Cost price(const Host& h, const Tier& t, int64_t tokens, const Option& o) {
+ Cost c;
+ c.opt = o;
+ c.chunks = o.chunk > 0 ? (tokens + o.chunk - 1) / o.chunk : 1;
+ const uint64_t unbacked = o.lend_bytes > o.lend_kept ? o.lend_bytes - o.lend_kept : 0;
+ c.file_set = (double) t.file_tier + (double) unbacked + (double) t.extra_file;
+ const double r = h.page_cache > 0 ? c.file_set / (double) h.page_cache : 1e9; // unbuffered: nothing is cached
+ c.miss = h.page_cache > 0 ? miss_share(r) : 1.0;
+ c.fits = h.page_cache > 0 && r <= 0.80;
+ c.read_bytes = (double) c.chunks * c.file_set * c.miss;
+ // a chunk streams every expert the GPU does not hold: the loan returns to that set for the length of the prompt
+ const double stream = std::max(0.0, (double) t.experts_total - (double) t.gpu_cache + (double) o.lend_bytes);
+ double ring_pen = 1.0;
+ if (o.ring_full > 16 && o.ring < o.ring_full)
+ ring_pen = 1.0 + h.ring_loss * (1.0 - (double) (o.ring - 16) / (double) (o.ring_full - 16));
+ const double compute = (double) tokens / h.compute_tok_s * ring_pen;
+ c.seconds = compute + (double) c.chunks * stream / (h.h2d_gbps * 1e9) + c.read_bytes / (h.ssd_gbps * 1e9);
+ return c;
+}
+
+struct Pick {
+ Cost best, baseline;
+ bool changed = false;
+};
+
+/// The option with the least predicted time. `baseline` (today's choice) is kept unless the pick saves at least
+/// `min_gain` of its time AND at least `min_save_s` seconds: a plan that fits the page cache is never touched, and
+/// near-ties keep the established behaviour.
+inline Pick choose(const Host& h, const Tier& t, int64_t tokens, const std::vector<Option>& options, size_t baseline,
+ double min_gain = 0.05, double min_save_s = 2.0) {
+ Pick p;
+ p.baseline = price(h, t, tokens, options.at(baseline));
+ p.best = p.baseline;
+ for (size_t i = 0; i < options.size(); ++i) {
+ if (i == baseline) continue;
+ const Cost c = price(h, t, tokens, options[i]);
+ if (c.seconds < p.best.seconds - 1e-9 ||
+ (std::abs(c.seconds - p.best.seconds) <= 1e-9 && c.opt.chunk > p.best.opt.chunk))
+ p.best = c;
+ }
+ p.changed = p.best.opt.chunk != p.baseline.opt.chunk || p.best.opt.ring != p.baseline.opt.ring;
+ if (p.changed && !(p.baseline.seconds - p.best.seconds >= min_gain * p.baseline.seconds &&
+ p.baseline.seconds - p.best.seconds >= min_save_s)) {
+ p.best = p.baseline;
+ p.changed = false;
+ }
+ return p;
+}
+
+/// One log line: what was chosen, why, and the reads it expects per prompt.
+inline std::string describe(const Pick& p, const Host& h, int64_t tokens) {
+ char b[768];
+ const Cost& c = p.best;
+ if (!p.changed) {
+ std::snprintf(b, sizeof b,
+ "strata: prefill plan: chunk %lld, ring %lld kept: file set %.1f GiB %s page cache %.1f GiB -> about %.1f GB "
+ "read from the drive per %lld-token prompt (%lld chunks)%s",
+ (long long) c.opt.chunk, (long long) c.opt.ring, c.file_set / kGiB,
+ c.fits ? "fits the" : "does not fit the", (double) h.page_cache / kGiB, c.read_bytes / 1e9,
+ (long long) tokens, (long long) c.chunks,
+ c.fits ? "" : "; no option does better (the set cannot fit: fewest passes win)");
+ } else {
+ const Cost& o = p.baseline;
+ std::snprintf(b, sizeof b,
+ "strata: prefill plan: chunk %lld -> %lld, ring %lld -> %lld: today's choice has a %.1f GiB file set "
+ "against a %.1f GiB page cache (about %.1f GB read from the drive per %lld-token prompt, %.0f s); this one "
+ "%.1f GiB, about %.1f GB read, %.0f s. --prefill N keeps your own chunk",
+ (long long) o.opt.chunk, (long long) c.opt.chunk, (long long) o.opt.ring, (long long) c.opt.ring,
+ o.file_set / kGiB, (double) h.page_cache / kGiB, o.read_bytes / 1e9, (long long) tokens, o.seconds,
+ c.file_set / kGiB, c.read_bytes / 1e9, c.seconds);
+ }
+ return b;
+}
+
+} // namespace strata::program::prefill_plan
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 3035bf166..70ee83a0b 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -76,6 +76,7 @@
#include "strata/prefill/share_rules.hpp"
#include "strata/platform/host_limits.hpp"
#include "strata/program/split_ring.hpp"
+#include "strata/program/prefill_plan.hpp"
#include "strata/program/draft_kv_plan.hpp"
#include "strata/program/message_boundary.hpp"
#include "strata/spec/draft_policy.hpp"
@@ -1349,6 +1350,24 @@ void disk_wait_threads(std::FILE* f) {
}
#endif
+// #2824: what the drive under `path` reads at, GB/s, for the prompt planner (it prices the file set a chunk re-reads). Linux:
+// rotational disk 0.15, NVMe 2.5, other 0.45 (SATA); elsewhere 1.5. STRATA_SSD_GBPS overrides. `why` says where it came from.
+double drive_read_gbps(const std::string& path, std::string& why) {
+ if (const char* e = std::getenv("STRATA_SSD_GBPS"); e != nullptr && std::atof(e) > 0.0) {
+ why = "STRATA_SSD_GBPS";
+ return std::atof(e);
+ }
+#if !defined(_WIN32)
+ std::string dev;
+ const int rot = on_rotational_disk(path, dev);
+ if (rot == 1) { why = dev + " is a rotational disk"; return 0.15; }
+ if (rot == 0 && dev.rfind("nvme", 0) == 0) { why = dev + " is NVMe"; return 2.5; }
+ if (rot == 0) { why = dev + " is a SATA-class SSD (assumed)"; return 0.45; }
+#endif
+ why = "assumed";
+ return 1.5;
+}
+
// --pipeline-windows: what the watchdog prints about the pipelined decode loop (set while it runs)
struct PlDiag {
strata::core::Verifier* v[2][2] = {};
@@ -1951,6 +1970,7 @@ static bool pcie_auto_split_ok() {
// median would still follow a disturbance that lasts through half the bursts). Each burst takes ~10 ms on an x16
// PCIe 4 link, so the probe takes no longer than the one 1 GiB burst did. `samples`, when given, gets every
// burst's reading for the log.
+double g_last_h2d_gbps = -1.0; // the last probe's reading, for the prompt planner (#2824)
double probe_pcie_h2d_gbps(std::string* samples = nullptr) {
constexpr size_t kBytes = 256ull << 20;
constexpr int kBursts = 4;
@@ -2006,6 +2026,7 @@ double probe_pcie_h2d_gbps(std::string* samples = nullptr) {
if (!ok) (void) cudaGetLastError();
cudaFree(d);
cudaFreeHost(h);
+ if (bw > 0.0) g_last_h2d_gbps = bw;
return bw;
}
@@ -7380,6 +7401,109 @@ int main(int argc, char** argv) {
}
return 0;
};
+ // ---- #2824: THE PROMPT PLAN PRICES THE HOST. The scans above ask whether a loan fits the GPU cache. But a lent
+ // slot's expert is not in VRAM for the length of the prompt: it comes from the RAM copy if the copy kept it, else from
+ // the model file, and a chunk streams every expert the GPU does not hold once per chunk. When that file set is bigger
+ // than the RAM the OS keeps file pages in, it cycles through the page cache and each chunk reads most of it again
+ // (2x RTX 3060, 32 GB, IQ2_XS, 29K prompt: 8192-token chunks read 28-45 GB, 4096 read ~1 GB). So the auto pick is
+ // refined by prefill_plan (include/strata/program/prefill_plan.hpp): the chunk x ring grid at or below the VRAM pick,
+ // each priced with the expert bytes its loan displaces without a RAM backup, the RAM left for the page cache, the
+ // measured PCIe rate and a drive-speed class, and the least predicted prompt time wins. A set that fits the page cache
+ // keeps today's pick untouched. Advisory: --prefill N and STRATA_PREFILL_RING are the operator's, and
+ // STRATA_PREFILL_HOST_PLAN=0 turns this off. No weights or experts change; only the chunk shape (rounding).
+ // fits_vram(c) whether every participant can lend chunk c at the ring currently set
+ // lend_of(c, bytes, slots) the cache bytes and slots the loan of chunk c takes at the ring currently set
+ // ring_full(c) the largest ring slot count on offer for chunk c
+ // restore() puts the ring state back as the VRAM pick left it
+ // Returns the chunk to use (== `chunk` when nothing changes); the ring state is set for it.
+ auto host_plan_chunk = [&](int64_t chunk, uint64_t gpu_cache_bytes, int64_t cache_slots,
+ const std::function<bool(int64_t)>& fits_vram,
+ const std::function<void(int64_t, uint64_t&, int64_t&)>& lend_of,
+ const std::function<int64_t(int64_t)>& ring_full,
+ const std::function<void()>& restore, int64_t tokens_hint = 0) -> int64_t {
+ namespace pp = strata::program::prefill_plan;
+ const char* off = std::getenv("STRATA_PREFILL_HOST_PLAN");
+ if (!o.prefill_auto || chunk <= 0 || srcp != &src || (off != nullptr && off[0] == '0') ||
+ !strata::prefill::Prefill::ring_bytes_enabled())
+ return chunk;
+ const auto& lay = strata::kernels::cpu::expert_layout();
+ uint64_t total = 0;
+ for (int64_t l = 0; l < g.n_layers; ++l) total += (uint64_t) lay.blob_bytes(l) * (uint64_t) g.n_expert;
+ if (total == 0 || gpu_cache_bytes >= total) return chunk; // the GPU holds every expert: no file tier
+ const auto hm = strata::core::sample_host_memory();
+ if (!hm) return chunk;
+ const uint64_t slack = 1200ull << 20; // MemAvailable overstates what the cache can hold (~1 GiB on the #2824 box)
+ pp::Host host;
+ host.page_cache = src.unbuffered() ? 0 : (hm->available > slack ? hm->available - slack : 0);
+ host.h2d_gbps = g_last_h2d_gbps > 0.0 ? g_last_h2d_gbps : 12.0;
+ std::string why;
+ host.ssd_gbps = drive_read_gbps(o.pack, why);
+ if (const char* e = std::getenv("STRATA_PREFILL_PLAN_TOKS"); e != nullptr && std::atof(e) > 0.0)
+ host.compute_tok_s = std::atof(e);
+ const uint64_t ram_copy = src.complement_ready() ? src.resident_bytes() : 0;
+ const int64_t kept_slots = src.complement_ready() ? src.resident_lent_slots() : 0;
+ const uint64_t slot_avg = cache_slots > 0 ? gpu_cache_bytes / (uint64_t) cache_slots : 0;
+ const uint64_t kept_bytes = (uint64_t) std::max<int64_t>(0, kept_slots) * slot_avg; // the copy's lend region
+ pp::Tier tier;
+ tier.experts_total = total;
+ tier.gpu_cache = gpu_cache_bytes;
+ const uint64_t ram_outside_gpu = ram_copy > kept_bytes ? ram_copy - kept_bytes : 0;
+ tier.file_tier = total > gpu_cache_bytes + ram_outside_gpu ? total - gpu_cache_bytes - ram_outside_gpu : 0;
+ tier.extra_file = o.ple_gguf.empty() ? 0 : (350ull << 20); // PLE rows the prompt touches (#2824: ~0.35 GiB)
+ const int64_t tokens = tokens_hint > 0 ? tokens_hint
+ : std::max<int64_t>(chunk, std::min<int64_t>(std::max<int64_t>(o.max_context, 4096), 32768));
+ std::vector<pp::Option> opts;
+ auto add = [&](int64_t c, int64_t ring_req) {
+ strata::prefill::Prefill::set_ring_budget((int) ring_req, 0);
+ if (!fits_vram(c)) return;
+ pp::Option op;
+ op.chunk = c;
+ op.ring = strata::prefill::Prefill::ring_slots_for(c);
+ op.ring_full = ring_full(c);
+ uint64_t lb = 0;
+ int64_t ls = 0;
+ lend_of(c, lb, ls);
+ op.lend_bytes = lb;
+ op.lend_kept = std::min<uint64_t>(lb, kept_bytes);
+ for (const pp::Option& e : opts) if (e.chunk == op.chunk && e.ring == op.ring) return;
+ opts.push_back(op);
+ };
+ // today's pick first, priced under the ring state it left
+ restore();
+ pp::Option base;
+ base.chunk = chunk;
+ base.ring = strata::prefill::Prefill::ring_slots_for(chunk);
+ base.ring_full = ring_full(chunk);
+ {
+ uint64_t lb = 0;
+ int64_t ls = 0;
+ lend_of(chunk, lb, ls);
+ base.lend_bytes = lb;
+ base.lend_kept = std::min<uint64_t>(lb, kept_bytes);
+ }
+ opts.push_back(base);
+ for (int64_t c = chunk; c >= 512; c -= 256) {
+ const int64_t full = ring_full(c);
+ add(c, (int) full);
+ if (full > 32) add(c, (int) (full / 2));
+ if (full > 16) add(c, 16);
+ }
+ pp::Pick pick = pp::choose(host, tier, tokens, opts, 0);
+ std::fprintf(stderr, "%s\n", pp::describe(pick, host, tokens).c_str());
+ std::fprintf(stderr, "strata: prefill plan inputs: %.1f GiB experts, %.1f GiB in the GPU caches, %.1f GiB RAM copy "
+ "(%lld lend slots kept), file tier %.1f GiB; %.1f GiB available for the page cache%s; PCIe %.1f GB/s; "
+ "drive %.2f GB/s (%s); a %lld-token prompt\n",
+ (double) total / pp::kGiB, (double) gpu_cache_bytes / pp::kGiB, (double) ram_copy / pp::kGiB,
+ (long long) kept_slots, (double) tier.file_tier / pp::kGiB, (double) host.page_cache / pp::kGiB,
+ src.unbuffered() ? " (the file tier reads unbuffered: none)" : "", host.h2d_gbps, host.ssd_gbps, why.c_str(),
+ (long long) tokens);
+ if (!pick.changed) {
+ restore();
+ return chunk;
+ }
+ strata::prefill::Prefill::set_ring_budget((int) pick.best.opt.ring, 0);
+ return pick.best.opt.chunk;
+ };
// ---- THE ELASTIC K/V (--kv-grow; vmm.hpp). Allocated for the whole --max-context up front, the K/V of a 262K
// context (3.35 GiB with the drafter's at int8) takes VRAM the expert cache could hold more experts in. Now the K/V pools hold physical memory only for the cells
// the requests reach. When a request needs more, slots just below the prompt path's loan give up their experts
@@ -8066,7 +8190,7 @@ int main(int argc, char** argv) {
if (c == 0 && cap < strata::prefill::Prefill::ring_default_slots()) c = probe(kRingMin);
return std::max(c, small); // never a smaller chunk than 0.1.39's
};
- const int64_t chunk = scan(nullptr);
+ int64_t chunk = scan(nullptr);
// #448: a small card in a layer split caps every stage's chunk (an RTX 3080's 512-slot cache held a
// 32 GB card's split to 512 tokens: prompts 6.2x slower, decode the same). Named when it bites, so the
// regression is one log line: each stage that cannot fund the chunk CUDA0 alone would read in.
@@ -8081,7 +8205,37 @@ int main(int argc, char** argv) {
small >= chunk ? 0 : (int) ring_room(chunk, nullptr, strata::prefill::Prefill::ring_cap_for(small)),
small);
}
- if (chunk > 0 && alone > chunk) {
+ // #2824: the VRAM pick, priced against the host (RAM left for the page cache, the displaced experts with no RAM
+ // backup, the drive): a smaller chunk / ring when the file set the prompt re-reads outgrows the page cache.
+ // The RAM copy is built by now, so its bytes and the lend slots it kept are known.
+ const int64_t chunk_vram = chunk;
+ if (chunk > 0 && o.prefill_auto && strata::prefill::Prefill::ring_bytes_enabled()) {
+ const int64_t small0 = old_pick(nullptr);
+ const int64_t cap0 = strata::prefill::Prefill::ring_cap_for(small0);
+ const int64_t ring0 = small0 >= chunk ? 0 : ring_room(chunk, nullptr, cap0);
+ auto restore_ring = [&] { strata::prefill::Prefill::set_ring_budget((int) ring0, small0); };
+ restore_ring();
+ uint64_t gc_bytes = 0;
+ int64_t gc_slots = 0;
+ for (const PfPart& p : pf_parts) {
+ gc_bytes += (uint64_t) std::max<int64_t>(0, (int64_t) p.cache->bytes());
+ gc_slots += p.cache->slots();
+ }
+ chunk = host_plan_chunk(
+ chunk, gc_bytes, gc_slots, [&](int64_t c) { return fits(c, true, nullptr); },
+ [&](int64_t c, uint64_t& bytes, int64_t& slots) {
+ bytes = 0;
+ slots = 0;
+ for (const PfPart& p : pf_parts) {
+ const int64_t k = part_slots(p, c);
+ if (k <= 0) continue;
+ slots += k;
+ bytes += part_bytes(p, (int32_t) (p.cache->slots() - k));
+ }
+ },
+ [&](int64_t c) { return ring_room(c, nullptr, cap0); }, restore_ring);
+ }
+ if (chunk > 0 && alone > chunk_vram) {
for (size_t i = 1; i < pf_parts.size(); ++i) {
const PfPart& p = pf_parts[i];
if (fits_one(p, alone, o.prefill_auto)) continue;
@@ -8096,7 +8250,7 @@ int main(int argc, char** argv) {
std::fprintf(stderr, "strata serve: WARNING: prompt chunk %lld tokens, not %lld: CUDA%d (%s) "
"has %lld expert-cache slots, and a %lld-token chunk borrows %lld of them "
"(it must keep 128%s) - prompts read slower than on CUDA0 alone (#448)\n",
- (long long) chunk, (long long) alone, dev, prop.name,
+ (long long) chunk_vram, (long long) alone, dev, prop.name,
(long long) p.cache->slots(), (long long) alone,
(long long) part_slots(p, alone), pct.c_str());
// the helper tiers start at CUDA1 without a split (and are enabled in order)
@@ -14981,7 +15135,24 @@ int main(int argc, char** argv) {
uint64_t borrow_bytes = 0;
if (!o.no_prefill_borrow && !host_res.empty() && d_res != nullptr) {
int64_t chunk = o.prefill_chunk;
+ const int64_t chunk_in = chunk;
int64_t k = plan_lend(chunk); // auto: the largest chunk that fits; fixed: halved to fit
+ if (k > 0 && o.prefill_auto && n_batched > 1024) { // #2824: priced against the host, for THIS prompt's length
+ const int64_t vram_chunk = chunk;
+ chunk = host_plan_chunk(
+ chunk, (uint64_t) std::max<int64_t>(0, (int64_t) xcache.bytes()), xcache.slots(),
+ [&](int64_t c) {
+ const int64_t kk = lend_slots(c);
+ return kk + 128 <= xcache.slots() && kk * 100 <= kAutoLendPct * xcache.slots();
+ },
+ [&](int64_t c, uint64_t& bytes, int64_t& slots) {
+ slots = lend_slots(c);
+ bytes = bytes_from_slots(slots);
+ },
+ [&](int64_t) { return strata::prefill::Prefill::ring_cap_for(4096); },
+ [&] { int64_t c2 = chunk_in; (void) plan_lend(c2); }, n_batched);
+ if (chunk != vram_chunk) k = lend_slots(chunk);
+ }
const int64_t request_sized = equal_chunk(n_batched, chunk);
if (k > 0 && request_sized < chunk) { // no bigger than this prompt segment needs
chunk = request_sized;
diff --git a/tests/program/prefill_plan_test.cpp b/tests/program/prefill_plan_test.cpp
new file mode 100644
index 000000000..baa4c4de2
--- /dev/null
+++ b/tests/program/prefill_plan_test.cpp
@@ -0,0 +1,96 @@
+// The prompt planner's host-memory model (#2824): no GPU, no model.
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+#include "strata/program/prefill_plan.hpp"
+
+using namespace strata::program::prefill_plan;
+
+static int failed = 0;
+#define CHECK(x) do { if (!(x)) { std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); ++failed; } } while (0)
+
+static uint64_t gib(double g) { return (uint64_t) (g * kGiB); }
+
+// options at the lend sizes the reporter's engine printed: 2048 1,688 slots 2.3 GiB, 4096 2.97K 4.0, 6144 5.8, 8192 7.5
+static std::vector<Option> issue_options(int64_t full_ring) {
+ std::vector<Option> v;
+ const struct { int64_t c; double g; } L[] = {{1024, 1.6}, {2048, 2.3}, {3072, 3.1}, {4096, 4.0}, {5120, 4.9}, {6144, 5.8}, {7168, 6.6}, {8192, 7.5}};
+ for (const auto& l : L) v.push_back({l.c, full_ring, full_ring, gib(l.g), 0});
+ return v;
+}
+
+int main() {
+ // #2824: 2x RTX 3060, 32 GB, IQ2_XS: the experts total 33.0 GiB, the GPU caches 13.84, the RAM copy 12.0 of the rest
+ Host h;
+ h.page_cache = gib(14.5);
+ h.h2d_gbps = 8.0;
+ h.ssd_gbps = 0.45;
+ Tier t;
+ t.experts_total = gib(33.0);
+ t.gpu_cache = gib(13.84);
+ t.file_tier = gib(7.18);
+ t.extra_file = gib(0.35);
+ const auto opts = issue_options(64);
+ // the file sets the reporter computed: 15.1 GiB at 8192, 13.3 at 6144, 11.6 at 4096
+ CHECK(std::abs(price(h, t, 29000, opts[7]).file_set / kGiB - 15.03) < 0.15);
+ CHECK(std::abs(price(h, t, 29000, opts[5]).file_set / kGiB - 13.33) < 0.15);
+ CHECK(std::abs(price(h, t, 29000, opts[3]).file_set / kGiB - 11.53) < 0.15);
+ // measured drive reads per 29K prompt: 8192: 28-45 GB, 4096: 0.9-1.4 GB, 6144: 1.2-2.8 GB
+ const Cost c8 = price(h, t, 28700, opts[7]), c6 = price(h, t, 28700, opts[5]), c4 = price(h, t, 28700, opts[3]);
+ std::printf("8192: set %.1f GiB miss %.2f reads %.1f GB %.0f s | 6144: %.1f GiB %.2f %.1f GB %.0f s | 4096: %.1f GiB %.2f %.1f GB %.0f s\n",
+ c8.file_set / kGiB, c8.miss, c8.read_bytes / 1e9, c8.seconds, c6.file_set / kGiB, c6.miss, c6.read_bytes / 1e9,
+ c6.seconds, c4.file_set / kGiB, c4.miss, c4.read_bytes / 1e9, c4.seconds);
+ CHECK(c8.read_bytes / 1e9 > 28 && c8.read_bytes / 1e9 < 46);
+ CHECK(c4.read_bytes == 0 && c4.fits);
+ CHECK(c6.read_bytes / 1e9 < 8);
+ // the planner leaves 8192 for the one that fits
+ const Pick p = choose(h, t, 28700, opts, 7);
+ std::printf("%s\n", describe(p, h, 28700).c_str());
+ CHECK(p.changed);
+ CHECK(p.best.opt.chunk >= 3072 && p.best.opt.chunk <= 4096);
+ CHECK(p.best.read_bytes < 3e9);
+
+ // a box where everything fits keeps today's choice
+ Host big = h;
+ big.page_cache = gib(40);
+ const Pick q = choose(big, t, 28700, opts, 7);
+ CHECK(!q.changed && q.best.opt.chunk == 8192 && q.best.read_bytes == 0);
+
+ // a box whose file set can never fit (8 GB card + 32 GB RAM): no refusal, no blind shrink: the fewest passes win
+ Tier hard;
+ hard.experts_total = gib(48);
+ hard.gpu_cache = gib(3.0);
+ hard.file_tier = gib(40);
+ hard.extra_file = gib(0.4);
+ Host small = h;
+ small.page_cache = gib(22);
+ std::vector<Option> o2;
+ for (int64_t c = 1024; c <= 8192; c += 1024) o2.push_back({c, 64, 64, gib(0.9 * (double) c / 1024.0), 0});
+ const Pick r = choose(small, hard, 28700, o2, 7);
+ std::printf("%s\n", describe(r, small, 28700).c_str());
+ CHECK(r.best.opt.chunk == 8192);
+ // and from a small today's choice it moves UP to fewer passes when nothing fits either way
+ const Pick r2 = choose(small, hard, 28700, o2, 1);
+ CHECK(r2.best.opt.chunk > 2048);
+
+ // unbuffered reads (no page cache): every chunk reads its set; the biggest chunk reads least
+ Host ub = h;
+ ub.page_cache = 0;
+ CHECK(price(ub, t, 28700, opts[7]).read_bytes < price(ub, t, 28700, opts[3]).read_bytes);
+
+ // lent experts the RAM copy keeps leave the file set: 8192 with all 7.5 GiB kept is 7.5 GiB smaller
+ Option kept = opts[7];
+ kept.lend_kept = kept.lend_bytes;
+ CHECK(price(h, t, 28700, kept).file_set < price(h, t, 28700, opts[7]).file_set - gib(7.4));
+ CHECK(price(h, t, 28700, kept).fits);
+
+ // a smaller ring only wins when it brings the set into the cache
+ Tier edge = t;
+ edge.file_tier = gib(6.2); // 8192 with the 64-slot ring: 6.2+7.5+.35 = 14.05 GiB, 0.97 of the cache
+ std::vector<Option> o3 = {{8192, 64, 64, gib(7.5), 0}, {8192, 16, 64, gib(7.0), 0}};
+ const Pick s = choose(h, edge, 28700, o3, 0);
+ std::printf("%s\n", describe(s, h, 28700).c_str());
+ std::printf("%s\n", failed ? "FAILED" : "prefill_plan_test: ok");
+ return failed ? 1 : 0;
+}