Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge perf-PR group cache (f-145-pp-cache)
Niko1221cbc6e3d8f822
diff --cc include/strata/kernels/verify_kernels.hppindex dab7c9fe3,567013a96..45bc70e1a--- a/include/strata/kernels/verify_kernels.hpp+++ b/include/strata/kernels/verify_kernels.hpp@@@ -74,17 -60,10 +74,21 @@@ void wait_flag_ge(const uint32_t* flag/// kernel costs ~2.4 us on a gfx1201 graph whatever it does. n <= 128 * k for any k (a strided loop).void wait_copy_i32_from_mapped(const uint32_t* flag, uint32_t value, int32_t* dst, const int32_t* src, int64_t n,void* stream);++/// Layer split, overlapped hand-off (STRATA_SPLIT_OVERLAP): dst (mapped) = [a (na) | b (nb) | c (nc)] with volatile+/// stores, then the LAST block to finish raises *flag = 1 (mapped) after a system fence, so the next stage's GPU can+/// start (wait_flag_ge(flag, 1)) without the host syncing this stage first. `counter` is one device word, 0 between+/// calls (the last block resets it). Volatile because on RDNA a plain store to mapped memory can sit in the L2.+/// `drop` (mapped, may be null): a test hook - nonzero leaves the flag down, as if the publish were lost.+void handoff_publish(float* dst, const float* a, int64_t na, const float* b, int64_t nb, const float* c, int64_t nc,+ uint32_t* counter, uint32_t* flag, void* stream, const uint32_t* drop = nullptr);+/// The reading stage's hand-off wait, bounded: spin until *flag >= value, or after timeout_ns (0: no bound) set+/// *err = 1 (mapped) and return, so the window's graph completes and the host fails the window cleanly.+void wait_handoff(const uint32_t* flag, uint32_t value, uint32_t* err, unsigned long long timeout_ns, void* stream);+ /// wait_flag_ge, also writing the GPU's clock (ns, gpu_stamp's) as the kernel starts into *t_in and as it returns+ /// into *t_out (either may be null): the verify window's PCIe-share balance, at no extra launch.+ void wait_flag_ge_stamped(const uint32_t* flag, uint32_t value, unsigned long long* t_in, unsigned long long* t_out,+ void* stream);/// the GPU's %globaltimer (ns) into buf[i] (a one-thread kernel: the verify window's stage profiler). Inside a PDL/// stretch (pdl.hpp) the stamp passes the early launch on, so a profiled window keeps the chain it measures.void gpu_stamp(unsigned long long* buf, int i, void* stream);diff --cc src/core/verify.cppindex 6c1c1e3e4,84da8ee5e..24c29a0a1--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@@ -571,9 -529,8 +571,9 @@@ Verifier::~Verifier()for (cudaEvent_t e : df_join_)if (e) cudaEventDestroy(e);if (arena_) cudaFree(arena_);+ if (hcount_d_) cudaFree(hcount_d_);void* hosts[] = {h_tok_, h_step_, h_pos_, h_commit_, h_ple_, h_out_, h_x_, h_ids_, h_w_, h_seq_, h_flag_, h_ymiss_,- h_flagA_, h_plan_, h_flagB_, h_plan_err_, h_nf_, herr_h_, hdrop_h_};- h_flagA_, h_plan_, h_flagB_, h_plan_err_, h_nf_, h_bal_};++ h_flagA_, h_plan_, h_flagB_, h_plan_err_, h_nf_, herr_h_, hdrop_h_, h_bal_};for (void* h : hosts)if (h) cudaFreeHost(h);}@@@ -2530,45 -2253,6 +2551,46 @@@ bool Verifier::adapt_gate_wait(int32_treturn true;}+bool Verifier::handoff_failed(std::string& err) const {+ if (herr_h_ == nullptr || *(const volatile uint32_t*) herr_h_ == 0) return false;+ err = "verify: layer split: the hand-off into layer " + std::to_string(lb_) + " did not arrive within " ++ std::to_string(g_split_wait_ns / 1000000ull) + " ms (STRATA_SPLIT_WAIT_MS); the window was not used. "+ "STRATA_SPLIT_OVERLAP=0 runs the serial order";+ trace_ev("HANDOFF-TIMEOUT", -1, lb_, (int64_t) (g_split_wait_ns / 1000000ull));+ return true;+}++bool Verifier::chain_overlap() const {+ return next_ != nullptr && hflag_out_h_ != nullptr && hflag_out_d_ != nullptr && hcount_d_ != nullptr &&+ next_->hflag_in_h_ == hflag_out_h_ && next_->device_ != device_ && !next_->ple_stage();+}++bool Verifier::prelaunch(int T, const int32_t* tokens, int64_t pos0, std::string& err) {+ const OnDevice on_device(device_);+ prelaunched_ = 0;+ cudaGraphExec_t exec_t = ar_off_ ? exec_nr_[T] : exec_[T]; // #871: the variant the previous stage refreshed+ if (T < 1 || T > max_t_ || exec_t == nullptr) { err = "verify: prelaunch before the window was captured"; return false; }+ if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }+ if (pos0 + T > ss_->qsa_states[ss_->qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }+ const Clock::time_point t0 = Clock::now();+ stage_inputs(T, tokens, pos0);+ if (trace_h_ != nullptr) std::memset(trace_h_, 0, trace_n_ * 8);+ std::atomic_thread_fence(std::memory_order_seq_cst);+ trace_ev("WINDOW-PRE", -1, -1, pos0 * 16 + T);+ ms_host += ms_since(t0);++ if (h_bal_ != nullptr) std::memset(h_bal_, 0, (size_t) bal_p_.size() * 4 * 8); // this window's stamps only+ if (ar_on() && h_plan_err_ != nullptr) *(volatile uint32_t*) h_plan_err_ = 0;+ // 0.1.43: the same gate run() applies before it launches (the adaptive tier's copies land first)+ if (adapt_gate_ != nullptr && adapt_gate_->active.load(std::memory_order_acquire) && ar_on() &&+ !adapt_gate_wait((int32_t) g_->n_layers, err)) return false;+ const cudaError_t le = cudaGraphLaunch(exec_t, cs_);+ trace_ev("LAUNCHED", -1, -1, (int64_t) le);+ if (le != cudaSuccess) { err = std::string("verify: launch: ") + cudaGetErrorString(le); return false; }+ (void) cudaStreamQuery(cs_);+ prelaunched_ = T;+ return true;+}+bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, void* user, int32_t* out,std::string& err) {#if defined(STRATA_GPU_METAL)@@@ -2584,21 -2268,12 +2606,22 @@@const ModelGeometry& g = *g_;SessionState& ss = *ss_;if (pos0 + T > ss.qsa_states[ss.qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }- if (pre_launch_ && !pre_launch_(cs_, err)) return false;- refresh_ar();+ // the overlapped split launched this stage's window already: keep the graph variant (#871) chosen for it then,+ // or this thread would serve the window as the other variant+ const bool pre = prelaunched_ == T;+ prelaunched_ = 0;++ if (!pre && pre_launch_ && !pre_launch_(cs_, err)) return false; // (a prelaunched window's was served with the first stage's)+ if (!pre) refresh_ar();if (!capture(T, err) || !capture_commit(err)) return false;+ const bool chain = chain_overlap();+ if (chain) { // capture the next stage's window here, on this thread, so the helper only stages and launches+ const OnDevice on_next(next_->device_);+ next_->refresh_ar(); // #871: the variant its prelaunch and its run() will both use+ if (!next_->capture(T, err) || !next_->capture_commit(err)) return false;+ }VDBG("captured; staging\n");const Clock::time_point t0 = Clock::now();- if (!staged_) stage_inputs(T, tokens, pos0);+ if (!pre && !staged_) stage_inputs(T, tokens, pos0);staged_ = false;const bool do_ple = ss.ple.ready() && ple_stage();uint32_t ple_rows[kVerifyMaxT * PLE_N_HEADS];@@@ -2611,64 -2286,36 +2634,65 @@@ss.ple.table->prefetch_rows(ple_rows + t * PLE_N_HEADS);}}- if (trace_h_ != nullptr) std::memset(trace_h_, 0, trace_n_ * 8); // #649: this window's breadcrumbs only- std::atomic_thread_fence(std::memory_order_seq_cst);- trace_ev("WINDOW", -1, -1, pos0 * 16 + T);- ms_host += ms_since(t0);- VDBG("staged; launching\n");- if (h_bal_ != nullptr) std::memset(h_bal_, 0, (size_t) bal_p_.size() * 4 * 8); // this window's stamps only- if (ar_on() && h_plan_err_ != nullptr) *(volatile uint32_t*) h_plan_err_ = 0;- // 0.1.43: with the all-resident graph the host serves no layer, so nothing can hold a layer back: every copy first- if (adapt_gate_ != nullptr && adapt_gate_->active.load(std::memory_order_acquire) && ar_on() &&- !adapt_gate_wait((int32_t) g.n_layers, err)) return false;#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)- // A blocking instrumented launch cannot also serve the graph's CPU doorbells.- static const bool threaded_launch = [] {- const char* value = std::getenv("STRATA_VERIFY_ASYNC_LAUNCH");- return value != nullptr && value[0] == '1';- }();- HostGraphLaunch launch(ar_off_ ? exec_nr_[T] : exec_[T], cs_, device_, threaded_launch,- [this] { (void) release_gpu_waits(5000); });- const cudaError_t le = launch.status();+ std::optional<HostGraphLaunch> launch_o; // (not set when the previous stage launched this window already)+#endif+ if (!pre) {+ if (trace_h_ != nullptr) std::memset(trace_h_, 0, trace_n_ * 8); // #649: this window's breadcrumbs only+ std::atomic_thread_fence(std::memory_order_seq_cst);+ trace_ev("WINDOW", -1, -1, pos0 * 16 + T);+ ms_host += ms_since(t0);+ VDBG("staged; launching\n");++ if (h_bal_ != nullptr) std::memset(h_bal_, 0, (size_t) bal_p_.size() * 4 * 8); // this window's stamps only+ if (ar_on() && h_plan_err_ != nullptr) *(volatile uint32_t*) h_plan_err_ = 0;+ // 0.1.43: with the all-resident graph the host serves no layer, so nothing can hold a layer back: every copy first+ if (adapt_gate_ != nullptr && adapt_gate_->active.load(std::memory_order_acquire) && ar_on() &&+ !adapt_gate_wait((int32_t) g.n_layers, err)) return false;+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)+ // A blocking instrumented launch cannot also serve the graph's CPU doorbells.+ static const bool threaded_launch = [] {+ const char* value = std::getenv("STRATA_VERIFY_ASYNC_LAUNCH");+ return value != nullptr && value[0] == '1';+ }();+ launch_o.emplace(ar_off_ ? exec_nr_[T] : exec_[T], cs_, device_, threaded_launch,+ [this] { (void) release_gpu_waits(5000); });+ const cudaError_t le = launch_o->status();#else- const cudaError_t le = cudaGraphLaunch(ar_off_ ? exec_nr_[T] : exec_[T], cs_);+ const cudaError_t le = cudaGraphLaunch(ar_off_ ? exec_nr_[T] : exec_[T], cs_);#endif- trace_ev("LAUNCHED", -1, -1, (int64_t) le);- if (le != cudaSuccess) { err = std::string("verify: launch: ") + cudaGetErrorString(le); return false; }+ trace_ev("LAUNCHED", -1, -1, (int64_t) le);+ if (le != cudaSuccess) { err = std::string("verify: launch: ") + cudaGetErrorString(le); return false; }#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)- (void) launch.query();+ (void) launch_o->query();#else- (void) cudaStreamQuery(cs_);+ (void) cudaStreamQuery(cs_);#endif- VDBG("launched\n");+ VDBG("launched\n");+ }+ // The overlapped split: the next stage's window is staged and launched on a helper thread while this thread+ // serves this stage's layers; its graph waits on the GPU for this stage's hand-off flag. The guard joins the+ // helper on every return, and on an early return (an error after the launch) releases the next stage's GPU+ // waits so no spin kernel outlives the window (#267).+ struct NextLaunch {+ Verifier* next = nullptr;+ std::thread th;+ bool ok = true;+ bool handed = false;+ std::string err;+ bool join(std::string& e) {+ if (th.joinable()) th.join();+ if (!ok) e = err;+ return ok;+ }+ ~NextLaunch() {+ if (th.joinable()) th.join();+ if (next != nullptr && !handed) (void) next->release_gpu_waits(5000);+ }+ } nl;+ if (chain) {+ nl.next = next_;+ nl.th = std::thread([&nl, T, tokens, pos0] { nl.ok = nl.next->prelaunch(T, tokens, pos0, nl.err); });+ }volatile uint32_t* const seq = h_seq_;volatile uint32_t* const flag = h_flag_;const int G = groups_[T] > 0 ? groups_[T] : 1;@@@ -3613,10 -3233,10 +3661,11 @@@ bool Verifier::batch_launch(int base, iif (b_running_) { err = "verify: batch_launch while this stage is busy"; return false; }int rows[8] = {};for (int t = 0; t < S && t < 8; ++t) rows[t] = base + t;+ if (pre_launch_ && !pre_launch_(cs_, err)) return false;if (!stage_batch(rows, S, base, tokens, pos, err)) return false;cudaError_t le = cudaGraphLaunch(exec_bm_[bkey(rows, S, base)], cs_);- if (le == cudaSuccess) le = cudaGraphLaunch(commit_bm_[bkey(rows, S, base)], cs_); // right behind it: every row is kept+ if (le == cudaSuccess && !batch_self_commits(rows, S)) // (else the window has kept its rows itself)+ le = cudaGraphLaunch(commit_bm_[bkey(rows, S, base)], cs_); // right behind it: every row is keptif (le != cudaSuccess) { err = std::string("verify: batch launch: ") + cudaGetErrorString(le); return false; }(void) cudaStreamQuery(cs_);if (ple_stage()) // the host's side of the commit (the hash's last two tokens)diff --cc src/kernels/cuda/verify_kernels.cuindex 0928c731a,e1cd4da42..854f375f5--- a/src/kernels/cuda/verify_kernels.cu+++ b/src/kernels/cuda/verify_kernels.cu@@@ -1169,65 -1142,16 +1179,74 @@@ __device__ __forceinline__ unsigned lon#endifreturn t;}++// the overlapped split's hand-off wait, bounded: spin until *flag >= value, or give up after timeout_ns (0: never),+// marking *err = 1 (mapped) so the host fails the window instead of using a stale hand-off. The window's later+// kernels still run (on whatever the hand-off buffer holds) so the graph completes and no stream is left hanging.+__global__ void wait_handoff_kernel(const volatile uint32_t* flag, uint32_t value, uint32_t* err,+ unsigned long long timeout_ns) {+ const unsigned long long t0 = timeout_ns ? strata_now_ns() : 0ull;+ while (*flag < value) {+ strata_spin_pause();+ if (timeout_ns && strata_now_ns() - t0 > timeout_ns) {+ *(volatile uint32_t*) err = 1u;+ __threadfence_system();+ return;+ }+ }+ __threadfence_system();+}++__global__ void handoff_publish_kernel(float* dst, const float* __restrict__ a, int64_t na, const float* __restrict__ b,+ int64_t nb, const float* __restrict__ c, int64_t nc, uint32_t* counter,+ uint32_t* flag, const volatile uint32_t* drop) {+ const int64_t n = na + nb + nc;+ volatile float* const out = dst;+ for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)+ out[i] = i < na ? a[i] : i < na + nb ? b[i - na] : c[i - na - nb];+ __threadfence_system(); // this thread's stores reach host memory before the block counts itself done+ __syncthreads();+ if (threadIdx.x == 0) {+ const uint32_t prev = atomicAdd(counter, 1u);+ if (prev == gridDim.x - 1) { // every block's payload is out: raise the next stage's flag+ *(volatile uint32_t*) counter = 0u;+ __threadfence_system();+ if (drop == nullptr || *drop == 0u) { // test hook (STRATA_TEST_HANDOFF_DROP): a lost publish+ *(volatile uint32_t*) flag = 1u;+ __threadfence_system();+ }+ }+ }+}+++ // wait_flag_ge, with the GPU's clock as it arrives (t_in) and as it leaves (t_out), either may be null+ __global__ void wait_flag_ge_stamped_kernel(const volatile uint32_t* flag, uint32_t value, volatile unsigned long long* t_in,+ volatile unsigned long long* t_out) {- if (t_in != nullptr) *t_in = now_ns();++ if (t_in != nullptr) *t_in = strata_now_ns();+ while (*flag < value) strata_spin_pause();+ __threadfence_system();- if (t_out != nullptr) *t_out = now_ns();++ if (t_out != nullptr) *t_out = strata_now_ns();+ }} // namespace+void wait_handoff(const uint32_t* flag, uint32_t value, uint32_t* err, unsigned long long timeout_ns, void* stream) {+ wait_handoff_kernel<<<1, 1, 0, (cudaStream_t) stream>>>(flag, value, err, timeout_ns);+ check("wait_handoff");+}++void handoff_publish(float* dst, const float* a, int64_t na, const float* b, int64_t nb, const float* c, int64_t nc,+ uint32_t* counter, uint32_t* flag, void* stream, const uint32_t* drop) {+ const int64_t n = na + nb + nc;+ if (n <= 0 || dst == nullptr || counter == nullptr || flag == nullptr) {+ std::fprintf(stderr, "handoff_publish: invalid arguments\n");+ std::exit(1);+ }+ const int64_t nb256 = (n + 255) / 256;+ const int blocks = (int) (nb256 < 64 ? nb256 : 64);+ handoff_publish_kernel<<<blocks, 256, 0, (cudaStream_t) stream>>>(dst, a, na, b, nb, c, nc, counter, flag, drop);+ check("handoff_publish");+}+namespace {// one thread per window entry: up to kVerifyMaxT tokens x 10 routed experts (80), so 128 (#646 had 64: a window of// 7+ tokens lost its last entries)diff --cc src/program/generate.cppindex 2c7f94923,efba5706c..a183866fd--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@@ -3220,28 -2956,17 +3281,36 @@@ int main(int argc, char** argv)std::fprintf(stderr, "strata generate: PCIe probe failed -> pcie_frac default %.2f\n", base);}}+ // STRATA_PCIE_BALANCE=1 (opt-in): each verify layer's PCIe count from measured costs (Verifier::set_pcie_balance)+ // instead of the probe's fixed share above - the CPU's speed sets the right share as much as the link's. IQ2_XS,+ // one AVX2 pool worker (emulated on an RTX 5090 + 9950X3D): 16.8-17.2 -> 12.8-13.0 ms a round; the 9950X3D's own+ // 16 workers: 12.2 either way. One GPU, no batch slots, no --pcie-frac.+ auto pcie_balance = [&]() {+ const char* e = std::getenv("STRATA_PCIE_BALANCE");+ return !pcie_given && native_pack && !multi_gpu && o.batch <= 0 && e != nullptr && std::strcmp(e, "1") == 0;+ };// the canonical Q2_0 pack's CPU kernels are AVX-512 only; a native pack runs on AVX2 CPUs as well- if (!native_pack) strata::kernels::cpu::cpu_require_expert_support();- else if (!strata::kernels::cpu::cpu_avx512_ok())+ if (!native_pack) {+ // #2347: an NVFP4 / i-quant / Unsloth pack IS a native pack and runs on an AVX2 CPU (its rows have AVX2+ // kernels). This refusal is the canonical Q2_0 pack's, reached when the --pack folder has no+ // native_experts.txt - a hand-written config pointing --pack at the wrong folder reads as "this CPU is too+ // old". Say so before the refusal, and where the file is when it sits one level down (a pack/ subfolder).+ if (!strata::kernels::cpu::cpu_features().usable()) {+ std::error_code ec;+ const std::filesystem::path pk(o.pack);+ std::fprintf(stderr, "strata generate: %s has no native_experts.txt, so it is read as the canonical Q2_0 "+ "pack, whose CPU kernel needs AVX-512.\n"+ " An NVFP4, i-quant or Unsloth pack has that file in the folder given as --pack "+ "and runs on an AVX2 CPU as well;\n",+ o.pack.c_str());+ if (std::filesystem::exists(pk / "pack" / "native_experts.txt", ec))+ std::fprintf(stderr, " it is in %s: use that folder as --pack.\n", (pk / "pack").generic_string().c_str());+ else+ std::fprintf(stderr, " if this is one of those packs, the folder is incomplete or the wrong one "+ "(setup writes it; run setup again).\n");+ }+ strata::kernels::cpu::cpu_require_expert_support();+ } else if (!strata::kernels::cpu::cpu_avx512_ok())std::fprintf(stderr, "strata generate: this CPU has no AVX-512: the expert kernels run on %s ""(multi-token for the i-quant gate/up rows)\n",!strata::kernels::cpu::cpu_avx2_ok()@@@ -8433,9 -8012,10 +8507,10 @@@for (int st = 0; st + 1 < n_stages; ++st) stage_ver_p(st, 1).set_next(&stage_ver_p(st + 1, 1), &split_drive_b);}ver.set_remote_expert_opt(remote_opt.get());+ if (pcie_balance() && n_stages == 1) ver.set_pcie_balance(&drive.d.pcie_model);if (batch_mtp) ver.set_batch_graph_limit(64); // --batch-mtp only: slot rotation makes many layouts (LRU); 0.1.39 keeps allif (!ver.init(wt, g, ss, vh, native_head.loaded() || prefill_head_alias_bound ? &native_head : nullptr,- batch_mtp ? strata::kernels::kVerifyMaxT : std::max(o.spec, o.batch), err) ||+ batch_mtp ? strata::kernels::kVerifyMaxT : std::max(o.spec, std::min(o.batch, (int) strata::kernels::kVerifyMaxT)), err) ||(use_mtp && !prefill_head_alias_bound && !mtp.bind(last_st && !mtp_first ? last_st->wt : wt, last_st ? &last_st->head : &native_head,pipe ? pl_mtp_R : ver.final_R_all(), err))) {std::fprintf(stderr, "strata serve: %s\n", err.c_str());@@@ -11419,9 -10881,8 +11607,9 @@@else if (key == "penalty_freq") req_penalty_freq = fv;else if (key == "penalty_present") req_penalty_present = fv;else if (key == "seed") req_seed = std::strtoull(tok.c_str() + eq + 1, nullptr, 10);- else if (key == "pcie_frac") req_pcie_frac = std::clamp((double) fv, 0.0, 1.0);+ else if (key == "pcie_frac") { req_pcie_frac = std::clamp((double) fv, 0.0, 1.0); req_pcie_set = true; }else if (key == "spec_min_p") req_spec_min_p = std::clamp((double) fv, 0.0, 1.0);+ else if (key == "pipe") req_pipe = std::max(0.0, (double) fv);else if (key == "aux_cpus") req_aux_cpus = std::atoi(tok.c_str() + eq + 1) != 0 ? 1 : 0;else if (key == "img_keys") {req_img_keys.clear();