Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Prefill planner: refit the miss share to two rigs (#2824 SATA 2x3060, P2003 NVMe cgroup sweep); read the drive class from the GGUF path

Niko1221d8471050bde2
diff --git a/include/strata/program/prefill_plan.hpp b/include/strata/program/prefill_plan.hpp
index 60d717416..2047f9807 100644
--- a/include/strata/program/prefill_plan.hpp
+++ b/include/strata/program/prefill_plan.hpp
@@ -64,15 +64,18 @@ struct Cost {
bool fits = false; ///< the set sits in the page cache with room to spare
};
-/// The share of a chunk's file set that comes from the drive, from r = file_set / page_cache. Fitted to #2824:
-/// r 0.80 (4096 chunks) read ~1 GB per prompt, r 0.92 (6144) 1.2-2.8 GB, r 1.04 (8192) 28-45 GB over four chunks
-/// (a miss share of 0.47-0.75). Below 0.80 the set stays cached; the ramp to 1.0 is the page cache's imperfect LRU on
-/// a cyclic scan (some pages survive); well past 1 nearly everything misses.
+/// The share of a chunk's file set that comes from the drive, from r = file_set / page_cache, with page_cache = the RAM
+/// available after the copy less a slack. Fitted to two rigs that disagree below r = 1 and agree above it:
+/// #2824 (2x RTX 3060, 32 GB, SATA, 4 chunk sizes): r 0.80 read ~1 GB per prompt, r 0.92 1.2-2.8 GB, r 1.04 a miss share of 0.47-0.75;
+/// P2003 (RTX 3060, IQ2_XS, cgroup 32G, 5 chunk sizes, 29K prompt): r 0.96 0.38, 1.00 0.44, 1.03 0.49, 1.07 0.61, 1.12 0.74.
+/// Below 0.85 the set stays cached; from there the share climbs (the page cache is never exactly the size it looks: the
+/// PLE shard and the startup reads keep pages in it), and past ~1.3 nearly every page misses.
inline double miss_share(double r) {
- if (r <= 0.80) return 0.0;
- if (r <= 1.00) return 0.15 * (r - 0.80) / 0.20;
- if (r <= 1.05) return 0.15 + 0.50 * (r - 1.00) / 0.05;
- return std::min(1.0, 0.65 + 0.35 * (r - 1.05) / 0.25);
+ if (r <= 0.85) return 0.0;
+ if (r <= 1.00) return 0.40 * (r - 0.85) / 0.15;
+ if (r <= 1.12) return 0.40 + 0.35 * (r - 1.00) / 0.12;
+ if (r <= 1.30) return 0.75 + 0.25 * (r - 1.12) / 0.18;
+ return 1.0;
}
inline Cost price(const Host& h, const Tier& t, int64_t tokens, const Option& o) {
@@ -83,7 +86,7 @@ inline Cost price(const Host& h, const Tier& t, int64_t tokens, const Option& o)
c.file_set = (double) t.file_tier + (double) unbacked + (double) t.extra_file;
const double r = h.page_cache > 0 ? c.file_set / (double) h.page_cache : 1e9; // unbuffered: nothing is cached
c.miss = h.page_cache > 0 ? miss_share(r) : 1.0;
- c.fits = h.page_cache > 0 && r <= 0.80;
+ c.fits = h.page_cache > 0 && r <= 0.85;
c.read_bytes = (double) c.chunks * c.file_set * c.miss;
// a chunk streams every expert the GPU does not hold: the loan returns to that set for the length of the prompt
const double stream = std::max(0.0, (double) t.experts_total - (double) t.gpu_cache + (double) o.lend_bytes);
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 70ee83a0b..b77e0e544 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -7437,7 +7437,7 @@ int main(int argc, char** argv) {
host.page_cache = src.unbuffered() ? 0 : (hm->available > slack ? hm->available - slack : 0);
host.h2d_gbps = g_last_h2d_gbps > 0.0 ? g_last_h2d_gbps : 12.0;
std::string why;
- host.ssd_gbps = drive_read_gbps(o.pack, why);
+ host.ssd_gbps = drive_read_gbps(o.native_preset.empty() ? o.pack : o.native_preset, why); // where the experts live
if (const char* e = std::getenv("STRATA_PREFILL_PLAN_TOKS"); e != nullptr && std::atof(e) > 0.0)
host.compute_tok_s = std::atof(e);
const uint64_t ram_copy = src.complement_ready() ? src.resident_bytes() : 0;
diff --git a/tests/program/prefill_plan_test.cpp b/tests/program/prefill_plan_test.cpp
index baa4c4de2..fc8e890f5 100644
--- a/tests/program/prefill_plan_test.cpp
+++ b/tests/program/prefill_plan_test.cpp
@@ -43,14 +43,30 @@ int main() {
c6.seconds, c4.file_set / kGiB, c4.miss, c4.read_bytes / 1e9, c4.seconds);
CHECK(c8.read_bytes / 1e9 > 28 && c8.read_bytes / 1e9 < 46);
CHECK(c4.read_bytes == 0 && c4.fits);
- CHECK(c6.read_bytes / 1e9 < 8);
+ CHECK(c6.read_bytes / 1e9 < 15); // measured 1.2-2.8 GB: the model is conservative here
// the planner leaves 8192 for the one that fits
const Pick p = choose(h, t, 28700, opts, 7);
std::printf("%s\n", describe(p, h, 28700).c_str());
CHECK(p.changed);
- CHECK(p.best.opt.chunk >= 3072 && p.best.opt.chunk <= 4096);
+ CHECK(p.best.opt.chunk >= 3072 && p.best.opt.chunk <= 5120);
CHECK(p.best.read_bytes < 3e9);
+ // P2003 (RTX 3060, IQ2_XS, cgroup 32G, NVMe): 8192 was the fastest of 2048/3328/4096/6144/8192 although it thrashes (measured 480 vs 391 tok/s at 4096).
+ // file sets 19.1 / 18.2 / 17.5 / 17.0 / 16.4 GiB against a 17.0 GiB cache; the fast drive makes the extra passes the dearer cost
+ Host nv;
+ nv.page_cache = gib(17.0);
+ nv.h2d_gbps = 25.6;
+ nv.ssd_gbps = 2.5;
+ Tier t2;
+ t2.experts_total = gib(33.0);
+ t2.gpu_cache = gib(6.0);
+ t2.file_tier = gib(15.0);
+ t2.extra_file = gib(0.35);
+ std::vector<Option> o4 = {{2048, 96, 96, gib(1.1), 0}, {3328, 96, 96, gib(1.63), 0}, {4096, 96, 96, gib(2.0), 0}, {6144, 96, 96, gib(2.9), 0}, {8192, 96, 96, gib(3.8), 0}};
+ const Pick n = choose(nv, t2, 28887, o4, 4);
+ std::puts(describe(n, nv, 28887).c_str());
+ CHECK(!n.changed);
+
// a box where everything fits keeps today's choice
Host big = h;
big.page_cache = gib(40);