Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge perf-PR group cache (f-145-pp-cache)

Niko1221cbc6e3d8f822
diff --cc include/strata/kernels/verify_kernels.hpp
index dab7c9fe3,567013a96..45bc70e1a
--- a/include/strata/kernels/verify_kernels.hpp
+++ b/include/strata/kernels/verify_kernels.hpp
@@@ -74,17 -60,10 +74,21 @@@ void wait_flag_ge(const uint32_t* flag
/// kernel costs ~2.4 us on a gfx1201 graph whatever it does. n <= 128 * k for any k (a strided loop).
void wait_copy_i32_from_mapped(const uint32_t* flag, uint32_t value, int32_t* dst, const int32_t* src, int64_t n,
void* stream);
+
+/// Layer split, overlapped hand-off (STRATA_SPLIT_OVERLAP): dst (mapped) = [a (na) | b (nb) | c (nc)] with volatile
+/// stores, then the LAST block to finish raises *flag = 1 (mapped) after a system fence, so the next stage's GPU can
+/// start (wait_flag_ge(flag, 1)) without the host syncing this stage first. `counter` is one device word, 0 between
+/// calls (the last block resets it). Volatile because on RDNA a plain store to mapped memory can sit in the L2.
+/// `drop` (mapped, may be null): a test hook - nonzero leaves the flag down, as if the publish were lost.
+void handoff_publish(float* dst, const float* a, int64_t na, const float* b, int64_t nb, const float* c, int64_t nc,
+ uint32_t* counter, uint32_t* flag, void* stream, const uint32_t* drop = nullptr);
+/// The reading stage's hand-off wait, bounded: spin until *flag >= value, or after timeout_ns (0: no bound) set
+/// *err = 1 (mapped) and return, so the window's graph completes and the host fails the window cleanly.
+void wait_handoff(const uint32_t* flag, uint32_t value, uint32_t* err, unsigned long long timeout_ns, void* stream);
+ /// wait_flag_ge, also writing the GPU's clock (ns, gpu_stamp's) as the kernel starts into *t_in and as it returns
+ /// into *t_out (either may be null): the verify window's PCIe-share balance, at no extra launch.
+ void wait_flag_ge_stamped(const uint32_t* flag, uint32_t value, unsigned long long* t_in, unsigned long long* t_out,
+ void* stream);
/// the GPU's %globaltimer (ns) into buf[i] (a one-thread kernel: the verify window's stage profiler). Inside a PDL
/// stretch (pdl.hpp) the stamp passes the early launch on, so a profiled window keeps the chain it measures.
void gpu_stamp(unsigned long long* buf, int i, void* stream);
diff --cc src/core/verify.cpp
index 6c1c1e3e4,84da8ee5e..24c29a0a1
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@@ -571,9 -529,8 +571,9 @@@ Verifier::~Verifier()
for (cudaEvent_t e : df_join_)
if (e) cudaEventDestroy(e);
if (arena_) cudaFree(arena_);
+ if (hcount_d_) cudaFree(hcount_d_);
void* hosts[] = {h_tok_, h_step_, h_pos_, h_commit_, h_ple_, h_out_, h_x_, h_ids_, h_w_, h_seq_, h_flag_, h_ymiss_,
- h_flagA_, h_plan_, h_flagB_, h_plan_err_, h_nf_, herr_h_, hdrop_h_};
- h_flagA_, h_plan_, h_flagB_, h_plan_err_, h_nf_, h_bal_};
++ h_flagA_, h_plan_, h_flagB_, h_plan_err_, h_nf_, herr_h_, hdrop_h_, h_bal_};
for (void* h : hosts)
if (h) cudaFreeHost(h);
}
@@@ -2530,45 -2253,6 +2551,46 @@@ bool Verifier::adapt_gate_wait(int32_t
return true;
}
+bool Verifier::handoff_failed(std::string& err) const {
+ if (herr_h_ == nullptr || *(const volatile uint32_t*) herr_h_ == 0) return false;
+ err = "verify: layer split: the hand-off into layer " + std::to_string(lb_) + " did not arrive within " +
+ std::to_string(g_split_wait_ns / 1000000ull) + " ms (STRATA_SPLIT_WAIT_MS); the window was not used. "
+ "STRATA_SPLIT_OVERLAP=0 runs the serial order";
+ trace_ev("HANDOFF-TIMEOUT", -1, lb_, (int64_t) (g_split_wait_ns / 1000000ull));
+ return true;
+}
+
+bool Verifier::chain_overlap() const {
+ return next_ != nullptr && hflag_out_h_ != nullptr && hflag_out_d_ != nullptr && hcount_d_ != nullptr &&
+ next_->hflag_in_h_ == hflag_out_h_ && next_->device_ != device_ && !next_->ple_stage();
+}
+
+bool Verifier::prelaunch(int T, const int32_t* tokens, int64_t pos0, std::string& err) {
+ const OnDevice on_device(device_);
+ prelaunched_ = 0;
+ cudaGraphExec_t exec_t = ar_off_ ? exec_nr_[T] : exec_[T]; // #871: the variant the previous stage refreshed
+ if (T < 1 || T > max_t_ || exec_t == nullptr) { err = "verify: prelaunch before the window was captured"; return false; }
+ if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }
+ if (pos0 + T > ss_->qsa_states[ss_->qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }
+ const Clock::time_point t0 = Clock::now();
+ stage_inputs(T, tokens, pos0);
+ if (trace_h_ != nullptr) std::memset(trace_h_, 0, trace_n_ * 8);
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ trace_ev("WINDOW-PRE", -1, -1, pos0 * 16 + T);
+ ms_host += ms_since(t0);
++ if (h_bal_ != nullptr) std::memset(h_bal_, 0, (size_t) bal_p_.size() * 4 * 8); // this window's stamps only
+ if (ar_on() && h_plan_err_ != nullptr) *(volatile uint32_t*) h_plan_err_ = 0;
+ // 0.1.43: the same gate run() applies before it launches (the adaptive tier's copies land first)
+ if (adapt_gate_ != nullptr && adapt_gate_->active.load(std::memory_order_acquire) && ar_on() &&
+ !adapt_gate_wait((int32_t) g_->n_layers, err)) return false;
+ const cudaError_t le = cudaGraphLaunch(exec_t, cs_);
+ trace_ev("LAUNCHED", -1, -1, (int64_t) le);
+ if (le != cudaSuccess) { err = std::string("verify: launch: ") + cudaGetErrorString(le); return false; }
+ (void) cudaStreamQuery(cs_);
+ prelaunched_ = T;
+ return true;
+}
+
bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, void* user, int32_t* out,
std::string& err) {
#if defined(STRATA_GPU_METAL)
@@@ -2584,21 -2268,12 +2606,22 @@@
const ModelGeometry& g = *g_;
SessionState& ss = *ss_;
if (pos0 + T > ss.qsa_states[ss.qsa_primary()].max_cells) { err = "verify: the window runs past the context"; return false; }
- if (pre_launch_ && !pre_launch_(cs_, err)) return false;
- refresh_ar();
+ // the overlapped split launched this stage's window already: keep the graph variant (#871) chosen for it then,
+ // or this thread would serve the window as the other variant
+ const bool pre = prelaunched_ == T;
+ prelaunched_ = 0;
++ if (!pre && pre_launch_ && !pre_launch_(cs_, err)) return false; // (a prelaunched window's was served with the first stage's)
+ if (!pre) refresh_ar();
if (!capture(T, err) || !capture_commit(err)) return false;
+ const bool chain = chain_overlap();
+ if (chain) { // capture the next stage's window here, on this thread, so the helper only stages and launches
+ const OnDevice on_next(next_->device_);
+ next_->refresh_ar(); // #871: the variant its prelaunch and its run() will both use
+ if (!next_->capture(T, err) || !next_->capture_commit(err)) return false;
+ }
VDBG("captured; staging\n");
const Clock::time_point t0 = Clock::now();
- if (!staged_) stage_inputs(T, tokens, pos0);
+ if (!pre && !staged_) stage_inputs(T, tokens, pos0);
staged_ = false;
const bool do_ple = ss.ple.ready() && ple_stage();
uint32_t ple_rows[kVerifyMaxT * PLE_N_HEADS];
@@@ -2611,64 -2286,36 +2634,65 @@@
ss.ple.table->prefetch_rows(ple_rows + t * PLE_N_HEADS);
}
}
- if (trace_h_ != nullptr) std::memset(trace_h_, 0, trace_n_ * 8); // #649: this window's breadcrumbs only
- std::atomic_thread_fence(std::memory_order_seq_cst);
- trace_ev("WINDOW", -1, -1, pos0 * 16 + T);
- ms_host += ms_since(t0);
- VDBG("staged; launching\n");
- if (h_bal_ != nullptr) std::memset(h_bal_, 0, (size_t) bal_p_.size() * 4 * 8); // this window's stamps only
- if (ar_on() && h_plan_err_ != nullptr) *(volatile uint32_t*) h_plan_err_ = 0;
- // 0.1.43: with the all-resident graph the host serves no layer, so nothing can hold a layer back: every copy first
- if (adapt_gate_ != nullptr && adapt_gate_->active.load(std::memory_order_acquire) && ar_on() &&
- !adapt_gate_wait((int32_t) g.n_layers, err)) return false;
#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)
- // A blocking instrumented launch cannot also serve the graph's CPU doorbells.
- static const bool threaded_launch = [] {
- const char* value = std::getenv("STRATA_VERIFY_ASYNC_LAUNCH");
- return value != nullptr && value[0] == '1';
- }();
- HostGraphLaunch launch(ar_off_ ? exec_nr_[T] : exec_[T], cs_, device_, threaded_launch,
- [this] { (void) release_gpu_waits(5000); });
- const cudaError_t le = launch.status();
+ std::optional<HostGraphLaunch> launch_o; // (not set when the previous stage launched this window already)
+#endif
+ if (!pre) {
+ if (trace_h_ != nullptr) std::memset(trace_h_, 0, trace_n_ * 8); // #649: this window's breadcrumbs only
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+ trace_ev("WINDOW", -1, -1, pos0 * 16 + T);
+ ms_host += ms_since(t0);
+ VDBG("staged; launching\n");
++ if (h_bal_ != nullptr) std::memset(h_bal_, 0, (size_t) bal_p_.size() * 4 * 8); // this window's stamps only
+ if (ar_on() && h_plan_err_ != nullptr) *(volatile uint32_t*) h_plan_err_ = 0;
+ // 0.1.43: with the all-resident graph the host serves no layer, so nothing can hold a layer back: every copy first
+ if (adapt_gate_ != nullptr && adapt_gate_->active.load(std::memory_order_acquire) && ar_on() &&
+ !adapt_gate_wait((int32_t) g.n_layers, err)) return false;
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)
+ // A blocking instrumented launch cannot also serve the graph's CPU doorbells.
+ static const bool threaded_launch = [] {
+ const char* value = std::getenv("STRATA_VERIFY_ASYNC_LAUNCH");
+ return value != nullptr && value[0] == '1';
+ }();
+ launch_o.emplace(ar_off_ ? exec_nr_[T] : exec_[T], cs_, device_, threaded_launch,
+ [this] { (void) release_gpu_waits(5000); });
+ const cudaError_t le = launch_o->status();
#else
- const cudaError_t le = cudaGraphLaunch(ar_off_ ? exec_nr_[T] : exec_[T], cs_);
+ const cudaError_t le = cudaGraphLaunch(ar_off_ ? exec_nr_[T] : exec_[T], cs_);
#endif
- trace_ev("LAUNCHED", -1, -1, (int64_t) le);
- if (le != cudaSuccess) { err = std::string("verify: launch: ") + cudaGetErrorString(le); return false; }
+ trace_ev("LAUNCHED", -1, -1, (int64_t) le);
+ if (le != cudaSuccess) { err = std::string("verify: launch: ") + cudaGetErrorString(le); return false; }
#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)
- (void) launch.query();
+ (void) launch_o->query();
#else
- (void) cudaStreamQuery(cs_);
+ (void) cudaStreamQuery(cs_);
#endif
- VDBG("launched\n");
+ VDBG("launched\n");
+ }
+ // The overlapped split: the next stage's window is staged and launched on a helper thread while this thread
+ // serves this stage's layers; its graph waits on the GPU for this stage's hand-off flag. The guard joins the
+ // helper on every return, and on an early return (an error after the launch) releases the next stage's GPU
+ // waits so no spin kernel outlives the window (#267).
+ struct NextLaunch {
+ Verifier* next = nullptr;
+ std::thread th;
+ bool ok = true;
+ bool handed = false;
+ std::string err;
+ bool join(std::string& e) {
+ if (th.joinable()) th.join();
+ if (!ok) e = err;
+ return ok;
+ }
+ ~NextLaunch() {
+ if (th.joinable()) th.join();
+ if (next != nullptr && !handed) (void) next->release_gpu_waits(5000);
+ }
+ } nl;
+ if (chain) {
+ nl.next = next_;
+ nl.th = std::thread([&nl, T, tokens, pos0] { nl.ok = nl.next->prelaunch(T, tokens, pos0, nl.err); });
+ }
volatile uint32_t* const seq = h_seq_;
volatile uint32_t* const flag = h_flag_;
const int G = groups_[T] > 0 ? groups_[T] : 1;
@@@ -3613,10 -3233,10 +3661,11 @@@ bool Verifier::batch_launch(int base, i
if (b_running_) { err = "verify: batch_launch while this stage is busy"; return false; }
int rows[8] = {};
for (int t = 0; t < S && t < 8; ++t) rows[t] = base + t;
+ if (pre_launch_ && !pre_launch_(cs_, err)) return false;
if (!stage_batch(rows, S, base, tokens, pos, err)) return false;
cudaError_t le = cudaGraphLaunch(exec_bm_[bkey(rows, S, base)], cs_);
- if (le == cudaSuccess) le = cudaGraphLaunch(commit_bm_[bkey(rows, S, base)], cs_); // right behind it: every row is kept
+ if (le == cudaSuccess && !batch_self_commits(rows, S)) // (else the window has kept its rows itself)
+ le = cudaGraphLaunch(commit_bm_[bkey(rows, S, base)], cs_); // right behind it: every row is kept
if (le != cudaSuccess) { err = std::string("verify: batch launch: ") + cudaGetErrorString(le); return false; }
(void) cudaStreamQuery(cs_);
if (ple_stage()) // the host's side of the commit (the hash's last two tokens)
diff --cc src/kernels/cuda/verify_kernels.cu
index 0928c731a,e1cd4da42..854f375f5
--- a/src/kernels/cuda/verify_kernels.cu
+++ b/src/kernels/cuda/verify_kernels.cu
@@@ -1169,65 -1142,16 +1179,74 @@@ __device__ __forceinline__ unsigned lon
#endif
return t;
}
+
+// the overlapped split's hand-off wait, bounded: spin until *flag >= value, or give up after timeout_ns (0: never),
+// marking *err = 1 (mapped) so the host fails the window instead of using a stale hand-off. The window's later
+// kernels still run (on whatever the hand-off buffer holds) so the graph completes and no stream is left hanging.
+__global__ void wait_handoff_kernel(const volatile uint32_t* flag, uint32_t value, uint32_t* err,
+ unsigned long long timeout_ns) {
+ const unsigned long long t0 = timeout_ns ? strata_now_ns() : 0ull;
+ while (*flag < value) {
+ strata_spin_pause();
+ if (timeout_ns && strata_now_ns() - t0 > timeout_ns) {
+ *(volatile uint32_t*) err = 1u;
+ __threadfence_system();
+ return;
+ }
+ }
+ __threadfence_system();
+}
+
+__global__ void handoff_publish_kernel(float* dst, const float* __restrict__ a, int64_t na, const float* __restrict__ b,
+ int64_t nb, const float* __restrict__ c, int64_t nc, uint32_t* counter,
+ uint32_t* flag, const volatile uint32_t* drop) {
+ const int64_t n = na + nb + nc;
+ volatile float* const out = dst;
+ for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n; i += (int64_t) gridDim.x * blockDim.x)
+ out[i] = i < na ? a[i] : i < na + nb ? b[i - na] : c[i - na - nb];
+ __threadfence_system(); // this thread's stores reach host memory before the block counts itself done
+ __syncthreads();
+ if (threadIdx.x == 0) {
+ const uint32_t prev = atomicAdd(counter, 1u);
+ if (prev == gridDim.x - 1) { // every block's payload is out: raise the next stage's flag
+ *(volatile uint32_t*) counter = 0u;
+ __threadfence_system();
+ if (drop == nullptr || *drop == 0u) { // test hook (STRATA_TEST_HANDOFF_DROP): a lost publish
+ *(volatile uint32_t*) flag = 1u;
+ __threadfence_system();
+ }
+ }
+ }
+}
++
+ // wait_flag_ge, with the GPU's clock as it arrives (t_in) and as it leaves (t_out), either may be null
+ __global__ void wait_flag_ge_stamped_kernel(const volatile uint32_t* flag, uint32_t value, volatile unsigned long long* t_in,
+ volatile unsigned long long* t_out) {
- if (t_in != nullptr) *t_in = now_ns();
++ if (t_in != nullptr) *t_in = strata_now_ns();
+ while (*flag < value) strata_spin_pause();
+ __threadfence_system();
- if (t_out != nullptr) *t_out = now_ns();
++ if (t_out != nullptr) *t_out = strata_now_ns();
+ }
} // namespace
+void wait_handoff(const uint32_t* flag, uint32_t value, uint32_t* err, unsigned long long timeout_ns, void* stream) {
+ wait_handoff_kernel<<<1, 1, 0, (cudaStream_t) stream>>>(flag, value, err, timeout_ns);
+ check("wait_handoff");
+}
+
+void handoff_publish(float* dst, const float* a, int64_t na, const float* b, int64_t nb, const float* c, int64_t nc,
+ uint32_t* counter, uint32_t* flag, void* stream, const uint32_t* drop) {
+ const int64_t n = na + nb + nc;
+ if (n <= 0 || dst == nullptr || counter == nullptr || flag == nullptr) {
+ std::fprintf(stderr, "handoff_publish: invalid arguments\n");
+ std::exit(1);
+ }
+ const int64_t nb256 = (n + 255) / 256;
+ const int blocks = (int) (nb256 < 64 ? nb256 : 64);
+ handoff_publish_kernel<<<blocks, 256, 0, (cudaStream_t) stream>>>(dst, a, na, b, nb, c, nc, counter, flag, drop);
+ check("handoff_publish");
+}
+
namespace {
// one thread per window entry: up to kVerifyMaxT tokens x 10 routed experts (80), so 128 (#646 had 64: a window of
// 7+ tokens lost its last entries)
diff --cc src/program/generate.cpp
index 2c7f94923,efba5706c..a183866fd
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@@ -3220,28 -2956,17 +3281,36 @@@ int main(int argc, char** argv)
std::fprintf(stderr, "strata generate: PCIe probe failed -> pcie_frac default %.2f\n", base);
}
}
+ // STRATA_PCIE_BALANCE=1 (opt-in): each verify layer's PCIe count from measured costs (Verifier::set_pcie_balance)
+ // instead of the probe's fixed share above - the CPU's speed sets the right share as much as the link's. IQ2_XS,
+ // one AVX2 pool worker (emulated on an RTX 5090 + 9950X3D): 16.8-17.2 -> 12.8-13.0 ms a round; the 9950X3D's own
+ // 16 workers: 12.2 either way. One GPU, no batch slots, no --pcie-frac.
+ auto pcie_balance = [&]() {
+ const char* e = std::getenv("STRATA_PCIE_BALANCE");
+ return !pcie_given && native_pack && !multi_gpu && o.batch <= 0 && e != nullptr && std::strcmp(e, "1") == 0;
+ };
// the canonical Q2_0 pack's CPU kernels are AVX-512 only; a native pack runs on AVX2 CPUs as well
- if (!native_pack) strata::kernels::cpu::cpu_require_expert_support();
- else if (!strata::kernels::cpu::cpu_avx512_ok())
+ if (!native_pack) {
+ // #2347: an NVFP4 / i-quant / Unsloth pack IS a native pack and runs on an AVX2 CPU (its rows have AVX2
+ // kernels). This refusal is the canonical Q2_0 pack's, reached when the --pack folder has no
+ // native_experts.txt - a hand-written config pointing --pack at the wrong folder reads as "this CPU is too
+ // old". Say so before the refusal, and where the file is when it sits one level down (a pack/ subfolder).
+ if (!strata::kernels::cpu::cpu_features().usable()) {
+ std::error_code ec;
+ const std::filesystem::path pk(o.pack);
+ std::fprintf(stderr, "strata generate: %s has no native_experts.txt, so it is read as the canonical Q2_0 "
+ "pack, whose CPU kernel needs AVX-512.\n"
+ " An NVFP4, i-quant or Unsloth pack has that file in the folder given as --pack "
+ "and runs on an AVX2 CPU as well;\n",
+ o.pack.c_str());
+ if (std::filesystem::exists(pk / "pack" / "native_experts.txt", ec))
+ std::fprintf(stderr, " it is in %s: use that folder as --pack.\n", (pk / "pack").generic_string().c_str());
+ else
+ std::fprintf(stderr, " if this is one of those packs, the folder is incomplete or the wrong one "
+ "(setup writes it; run setup again).\n");
+ }
+ strata::kernels::cpu::cpu_require_expert_support();
+ } else if (!strata::kernels::cpu::cpu_avx512_ok())
std::fprintf(stderr, "strata generate: this CPU has no AVX-512: the expert kernels run on %s "
"(multi-token for the i-quant gate/up rows)\n",
!strata::kernels::cpu::cpu_avx2_ok()
@@@ -8433,9 -8012,10 +8507,10 @@@
for (int st = 0; st + 1 < n_stages; ++st) stage_ver_p(st, 1).set_next(&stage_ver_p(st + 1, 1), &split_drive_b);
}
ver.set_remote_expert_opt(remote_opt.get());
+ if (pcie_balance() && n_stages == 1) ver.set_pcie_balance(&drive.d.pcie_model);
if (batch_mtp) ver.set_batch_graph_limit(64); // --batch-mtp only: slot rotation makes many layouts (LRU); 0.1.39 keeps all
if (!ver.init(wt, g, ss, vh, native_head.loaded() || prefill_head_alias_bound ? &native_head : nullptr,
- batch_mtp ? strata::kernels::kVerifyMaxT : std::max(o.spec, o.batch), err) ||
+ batch_mtp ? strata::kernels::kVerifyMaxT : std::max(o.spec, std::min(o.batch, (int) strata::kernels::kVerifyMaxT)), err) ||
(use_mtp && !prefill_head_alias_bound && !mtp.bind(last_st && !mtp_first ? last_st->wt : wt, last_st ? &last_st->head : &native_head,
pipe ? pl_mtp_R : ver.final_R_all(), err))) {
std::fprintf(stderr, "strata serve: %s\n", err.c_str());
@@@ -11419,9 -10881,8 +11607,9 @@@
else if (key == "penalty_freq") req_penalty_freq = fv;
else if (key == "penalty_present") req_penalty_present = fv;
else if (key == "seed") req_seed = std::strtoull(tok.c_str() + eq + 1, nullptr, 10);
- else if (key == "pcie_frac") req_pcie_frac = std::clamp((double) fv, 0.0, 1.0);
+ else if (key == "pcie_frac") { req_pcie_frac = std::clamp((double) fv, 0.0, 1.0); req_pcie_set = true; }
else if (key == "spec_min_p") req_spec_min_p = std::clamp((double) fv, 0.0, 1.0);
+ else if (key == "pipe") req_pipe = std::max(0.0, (double) fv);
else if (key == "aux_cpus") req_aux_cpus = std::atoi(tok.c_str() + eq + 1) != 0 ? 1 : 0;
else if (key == "img_keys") {
req_img_keys.clear();