Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
sycl: PcieAuto on SYCL, opt-in STRATA_PCIE_FRAC_DEFAULT=auto (single card, native pack): the PCIe share refined from the CPU pool's speed in the first decode windows
Niko1221c1a5b13fdf04
diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cppindex d11f2f2d7..a03c11f11 100644--- a/sycl/src/program/generate.cpp+++ b/sycl/src/program/generate.cpp@@ -640,6 +640,8 @@ struct Options {/// Plan v0.3 P6: the share (0..1) of each layer's distinct missed experts the GPU reads over PCIe from the/// pinned arena while the CPU computes the rest (verify windows).double pcie_frac = -1.0; ///< < 0: the model's default (0.2 direct for the Q2_0 pack, 0.55 DMA for native packs)+ bool pcie_frac_auto = false; ///< STRATA_PCIE_FRAC_DEFAULT=auto, single card, no --pcie-frac: the share is refined in the first decode windows (PcieAuto)+ double pcie_probe_gbps = 0.0; ///< the start-up probe's reading (for PcieAuto)std::string pcie_mode = "auto"; ///< auto | dma | kernel | direct/// Plan v0.3 P6: every `adapt_every` rounds, swap up to `adapt_swaps` of the most-routed missing experts into/// the VRAM tier in place of the least-routed resident ones (decayed counts). 0 = static residency.@@ -1628,6 +1630,46 @@ double pcie_frac_for_gbps(double gbps, double base) {return gbps <= 0.0 ? base : base * std::min(1.0, gbps / 20.0);}+// 0.1.42 (CUDA) / 0.1.48 (SYCL, opt-in STRATA_PCIE_FRAC_DEFAULT=auto): the share refined from what the CPU pool measures in+// the first decode windows. The balance share of the missed experts is link / (link + 1.85 x CPU rate); the link is the+// start-up probe, the CPU rate the expert bytes over the pool's wall time per missed expert (ms_run / multi_misses).+// Three updates at most (after 48, then 96 and 96 more windows), each rounded to 0.05.+struct PcieAuto {+ static constexpr double kPoolFactor = 1.85;+ bool on = false;+ double link_gbps = 0.0, blob_bytes = 0.0;+ double mark_ms = 0.0;+ int64_t mark_jobs = 0;+ double acc_ms = 0.0;+ int64_t acc_jobs = 0;+ int acc_win = 0, updates = 0;+ void mark(double ms_run, int64_t jobs) { mark_ms = ms_run; mark_jobs = jobs; }+ /// After a decode window: the new share, or -1 while there is nothing to change. `tc_us`/`rate` get the reading.+ double window(double ms_run, int64_t jobs, double cur, double* tc_us, double* rate) {+ if (!on) return -1.0;+ acc_ms += ms_run - mark_ms;+ acc_jobs += jobs - mark_jobs;+ mark(ms_run, jobs);+ ++acc_win;+ if (acc_win < (updates == 0 ? 48 : 96)) return -1.0;+ if (acc_jobs < 300) { // too few missed experts to read the pool's speed (everything hits): look again+ if (acc_win >= 2000) on = false;+ return -1.0;+ }+ const double tc = acc_ms * 1000.0 / (double) acc_jobs;+ acc_ms = 0.0;+ acc_jobs = 0;+ acc_win = 0;+ if (++updates >= 3) on = false;+ if (!(tc > 1.0) || !(blob_bytes > 0.0) || !(link_gbps > 0.0)) return -1.0;+ const double r = blob_bytes / (tc * 1e-6) / 1e9;+ const double f = std::min(0.60, std::max(0.10, std::round(link_gbps / (link_gbps + kPoolFactor * r) * 20.0) / 20.0));+ *tc_us = tc;+ *rate = r;+ return std::fabs(f - cur) < 0.025 ? -1.0 : f;+ }+};+} // namespace// #1425 #1549 #1629: --ple-io direct (the default) does unbuffered random reads of the n-gram table; on some drives@@ -2602,6 +2644,11 @@ int main(int argc, char **argv) try {o.pcie_frac = pcie_frac_for_gbps(bw, base);std::fprintf(stderr, "strata generate: PCIe probe: %.1f GB/s host->device (best of %s) -> pcie_frac %.2f ""(default %.2f)\n", bw, bursts.c_str(), o.pcie_frac, base);+ { // PcieAuto on SYCL: opt-in (STRATA_PCIE_FRAC_DEFAULT=auto), one card, not the xe NO_HOST path+ const char* pfd = std::getenv("STRATA_PCIE_FRAC_DEFAULT");+ o.pcie_frac_auto = pfd != nullptr && std::strcmp(pfd, "auto") == 0 && o.layer_split.empty();+ o.pcie_probe_gbps = bw;+ }} else {o.pcie_frac = base;std::fprintf(stderr, "strata generate: PCIe probe failed -> pcie_frac default %.2f\n", base);@@ -7887,6 +7934,29 @@ int main(int argc, char **argv) try {std::all_of(host_res.begin(), host_res.end(), [](int32_t r) { return r >= 0; });if (o.adapt_every > 0 && o.adapt_swaps > 0 && !all_experts_resident)drive.d.usage.assign((size_t) (g.n_layers * g.n_expert), 0.0f);+ PcieAuto pcie_auto;+ if (o.pcie_frac_auto && !all_experts_resident && o.batch == 0 && n_stages == 1) {+ const auto& pl = strata::kernels::cpu::expert_layout();+ double sum = 0.0;+ for (int64_t l = 0; l < g.n_layers; ++l) sum += (double) pl.blob_bytes(l);+ pcie_auto.on = true;+ pcie_auto.link_gbps = o.pcie_probe_gbps;+ pcie_auto.blob_bytes = g.n_layers > 0 ? sum / (double) g.n_layers : 0.0;+ }+ auto pcie_auto_step = [&](double& req_pcie_frac) {+ if (!pcie_auto.on) return;+ double tc_us = 0.0, rate = 0.0;+ const double nf = req_pcie_frac == o.pcie_frac+ ? pcie_auto.window(drive.d.ms_run, drive.d.multi_misses, o.pcie_frac, &tc_us, &rate)+ : (pcie_auto.mark(drive.d.ms_run, drive.d.multi_misses), -1.0);+ if (nf < 0.0) return;+ std::fprintf(stderr, "strata serve: PCIe share: the CPU pool takes %.0f us per missed expert (%.1f GB/s of expert bytes), "+ "the link %.1f GB/s -> pcie_frac %.2f (was %.2f; --pcie-frac N fixes it)\n",+ tc_us, rate, pcie_auto.link_gbps, nf, o.pcie_frac);+ o.pcie_frac = nf;+ req_pcie_frac = nf;+ drive.d.pcie_num = std::max(0, std::min(256, (int) (nf * 256.0 + 0.5)));+ };// #477 --expert-profile-save: what the adaptive tier learned, kept across restarts (opt-in; off: `heat` stays// empty and nothing below runs). It needs the adaptive tier's counts and the residency table.std::vector<double> heat;@@ -10639,6 +10709,7 @@ int main(int argc, char **argv) try {return p != nullptr ? std::fopen(p, "ab") : nullptr;}();const int64_t decode_hits0 = drive.d.cache_hits;+ pcie_auto.mark(drive.d.ms_run, drive.d.multi_misses); // the prompt's windows are not the decode's// CS-T: the RAM and file tiers of this request (the mmap source; 0 with the arena)const int64_t ram0 = src.ram_reads(), files0 = src.file_reads();const uint64_t file_bytes0 = src.file_read_bytes();@@ -11499,6 +11570,7 @@ int main(int argc, char **argv) try {dt_run += msd(tw0, tw1); dt_commit += msd(tw1, tw2); dt_draft += msd(tw2, tw3);++dec_windows; dec_T += T;}+ pcie_auto_step(req_pcie_frac); // PcieAuto (opt-in): the share refined from the CPU pool's speedif (adapt_thr.joinable()) adapt_thr.join();if (!adapt_ok) {std::printf("ERR an adaptive refill failed\n");