Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

sycl: PcieAuto on SYCL, opt-in STRATA_PCIE_FRAC_DEFAULT=auto (single card, native pack): the PCIe share refined from the CPU pool's speed in the first decode windows

Niko1221c1a5b13fdf04
diff --git a/sycl/src/program/generate.cpp b/sycl/src/program/generate.cpp
index d11f2f2d7..a03c11f11 100644
--- a/sycl/src/program/generate.cpp
+++ b/sycl/src/program/generate.cpp
@@ -640,6 +640,8 @@ struct Options {
/// Plan v0.3 P6: the share (0..1) of each layer's distinct missed experts the GPU reads over PCIe from the
/// pinned arena while the CPU computes the rest (verify windows).
double pcie_frac = -1.0; ///< < 0: the model's default (0.2 direct for the Q2_0 pack, 0.55 DMA for native packs)
+ bool pcie_frac_auto = false; ///< STRATA_PCIE_FRAC_DEFAULT=auto, single card, no --pcie-frac: the share is refined in the first decode windows (PcieAuto)
+ double pcie_probe_gbps = 0.0; ///< the start-up probe's reading (for PcieAuto)
std::string pcie_mode = "auto"; ///< auto | dma | kernel | direct
/// Plan v0.3 P6: every `adapt_every` rounds, swap up to `adapt_swaps` of the most-routed missing experts into
/// the VRAM tier in place of the least-routed resident ones (decayed counts). 0 = static residency.
@@ -1628,6 +1630,46 @@ double pcie_frac_for_gbps(double gbps, double base) {
return gbps <= 0.0 ? base : base * std::min(1.0, gbps / 20.0);
}
+// 0.1.42 (CUDA) / 0.1.48 (SYCL, opt-in STRATA_PCIE_FRAC_DEFAULT=auto): the share refined from what the CPU pool measures in
+// the first decode windows. The balance share of the missed experts is link / (link + 1.85 x CPU rate); the link is the
+// start-up probe, the CPU rate the expert bytes over the pool's wall time per missed expert (ms_run / multi_misses).
+// Three updates at most (after 48, then 96 and 96 more windows), each rounded to 0.05.
+struct PcieAuto {
+ static constexpr double kPoolFactor = 1.85;
+ bool on = false;
+ double link_gbps = 0.0, blob_bytes = 0.0;
+ double mark_ms = 0.0;
+ int64_t mark_jobs = 0;
+ double acc_ms = 0.0;
+ int64_t acc_jobs = 0;
+ int acc_win = 0, updates = 0;
+ void mark(double ms_run, int64_t jobs) { mark_ms = ms_run; mark_jobs = jobs; }
+ /// After a decode window: the new share, or -1 while there is nothing to change. `tc_us`/`rate` get the reading.
+ double window(double ms_run, int64_t jobs, double cur, double* tc_us, double* rate) {
+ if (!on) return -1.0;
+ acc_ms += ms_run - mark_ms;
+ acc_jobs += jobs - mark_jobs;
+ mark(ms_run, jobs);
+ ++acc_win;
+ if (acc_win < (updates == 0 ? 48 : 96)) return -1.0;
+ if (acc_jobs < 300) { // too few missed experts to read the pool's speed (everything hits): look again
+ if (acc_win >= 2000) on = false;
+ return -1.0;
+ }
+ const double tc = acc_ms * 1000.0 / (double) acc_jobs;
+ acc_ms = 0.0;
+ acc_jobs = 0;
+ acc_win = 0;
+ if (++updates >= 3) on = false;
+ if (!(tc > 1.0) || !(blob_bytes > 0.0) || !(link_gbps > 0.0)) return -1.0;
+ const double r = blob_bytes / (tc * 1e-6) / 1e9;
+ const double f = std::min(0.60, std::max(0.10, std::round(link_gbps / (link_gbps + kPoolFactor * r) * 20.0) / 20.0));
+ *tc_us = tc;
+ *rate = r;
+ return std::fabs(f - cur) < 0.025 ? -1.0 : f;
+ }
+};
+
} // namespace
// #1425 #1549 #1629: --ple-io direct (the default) does unbuffered random reads of the n-gram table; on some drives
@@ -2602,6 +2644,11 @@ int main(int argc, char **argv) try {
o.pcie_frac = pcie_frac_for_gbps(bw, base);
std::fprintf(stderr, "strata generate: PCIe probe: %.1f GB/s host->device (best of %s) -> pcie_frac %.2f "
"(default %.2f)\n", bw, bursts.c_str(), o.pcie_frac, base);
+ { // PcieAuto on SYCL: opt-in (STRATA_PCIE_FRAC_DEFAULT=auto), one card, not the xe NO_HOST path
+ const char* pfd = std::getenv("STRATA_PCIE_FRAC_DEFAULT");
+ o.pcie_frac_auto = pfd != nullptr && std::strcmp(pfd, "auto") == 0 && o.layer_split.empty();
+ o.pcie_probe_gbps = bw;
+ }
} else {
o.pcie_frac = base;
std::fprintf(stderr, "strata generate: PCIe probe failed -> pcie_frac default %.2f\n", base);
@@ -7887,6 +7934,29 @@ int main(int argc, char **argv) try {
std::all_of(host_res.begin(), host_res.end(), [](int32_t r) { return r >= 0; });
if (o.adapt_every > 0 && o.adapt_swaps > 0 && !all_experts_resident)
drive.d.usage.assign((size_t) (g.n_layers * g.n_expert), 0.0f);
+ PcieAuto pcie_auto;
+ if (o.pcie_frac_auto && !all_experts_resident && o.batch == 0 && n_stages == 1) {
+ const auto& pl = strata::kernels::cpu::expert_layout();
+ double sum = 0.0;
+ for (int64_t l = 0; l < g.n_layers; ++l) sum += (double) pl.blob_bytes(l);
+ pcie_auto.on = true;
+ pcie_auto.link_gbps = o.pcie_probe_gbps;
+ pcie_auto.blob_bytes = g.n_layers > 0 ? sum / (double) g.n_layers : 0.0;
+ }
+ auto pcie_auto_step = [&](double& req_pcie_frac) {
+ if (!pcie_auto.on) return;
+ double tc_us = 0.0, rate = 0.0;
+ const double nf = req_pcie_frac == o.pcie_frac
+ ? pcie_auto.window(drive.d.ms_run, drive.d.multi_misses, o.pcie_frac, &tc_us, &rate)
+ : (pcie_auto.mark(drive.d.ms_run, drive.d.multi_misses), -1.0);
+ if (nf < 0.0) return;
+ std::fprintf(stderr, "strata serve: PCIe share: the CPU pool takes %.0f us per missed expert (%.1f GB/s of expert bytes), "
+ "the link %.1f GB/s -> pcie_frac %.2f (was %.2f; --pcie-frac N fixes it)\n",
+ tc_us, rate, pcie_auto.link_gbps, nf, o.pcie_frac);
+ o.pcie_frac = nf;
+ req_pcie_frac = nf;
+ drive.d.pcie_num = std::max(0, std::min(256, (int) (nf * 256.0 + 0.5)));
+ };
// #477 --expert-profile-save: what the adaptive tier learned, kept across restarts (opt-in; off: `heat` stays
// empty and nothing below runs). It needs the adaptive tier's counts and the residency table.
std::vector<double> heat;
@@ -10639,6 +10709,7 @@ int main(int argc, char **argv) try {
return p != nullptr ? std::fopen(p, "ab") : nullptr;
}();
const int64_t decode_hits0 = drive.d.cache_hits;
+ pcie_auto.mark(drive.d.ms_run, drive.d.multi_misses); // the prompt's windows are not the decode's
// CS-T: the RAM and file tiers of this request (the mmap source; 0 with the arena)
const int64_t ram0 = src.ram_reads(), files0 = src.file_reads();
const uint64_t file_bytes0 = src.file_read_bytes();
@@ -11499,6 +11570,7 @@ int main(int argc, char **argv) try {
dt_run += msd(tw0, tw1); dt_commit += msd(tw1, tw2); dt_draft += msd(tw2, tw3);
++dec_windows; dec_T += T;
}
+ pcie_auto_step(req_pcie_frac); // PcieAuto (opt-in): the share refined from the CPU pool's speed
if (adapt_thr.joinable()) adapt_thr.join();
if (!adapt_ok) {
std::printf("ERR an adaptive refill failed\n");