Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Merge f-027-gh-eng-k1 (#3867 #3751 #3453 #3943 #3820 #3752 #3455: Pascal and sm_89 opt-in kernel ports)

Niko1221551f1cdc301a
diff --cc src/core/expert_cache.cpp
index f499378c3,72523b9cd..d670c8da7
--- a/src/core/expert_cache.cpp
+++ b/src/core/expert_cache.cpp
@@@ -22,65 -22,19 +22,76 @@@
#include <utility>
#include <cstring>
+// (not in a SYCL build: its CUDA names are a compatibility layer without the PCI ids NVML is matched by)
+#if defined(_WIN32) && !defined(STRATA_EC_NO_VMM) && !defined(SYCL_LANGUAGE_VERSION)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#define STRATA_EC_NVML_FREE 1
+#endif
+
namespace strata::core {
+#if defined(STRATA_EC_NVML_FREE)
+namespace {
+
+// STRATA_VRAM_FREE=device (opt-in, Windows + NVIDIA). Under WDDM cudaMemGetInfo answers for THIS process: memory
+// another process holds on the card (the image encoder, a browser, another engine) is not in its free figure.
+// Measured on an RTX 5090, driver 610.88: with 6 GiB written by another process, cudaMemGetInfo in a second process
+// still read 30,994 MiB free where NVML read 24,348. NVML (nvml.dll, installed with every driver) counts the whole
+// card. The library is loaded from the system directory only, at first use; false when it cannot answer.
+bool nvml_device_free(size_t& out) {
+ struct Memory { unsigned long long total, free, used; };
+ typedef int (*InitFn)(void);
+ typedef int (*ByBusFn)(const char*, void**);
+ typedef int (*MemFn)(void*, Memory*);
+ struct Lib {
+ ByBusFn by_bus = nullptr;
+ MemFn mem = nullptr;
+ Lib() {
+ HMODULE h = LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32);
+ if (h == nullptr) return;
+ const auto init = (InitFn) (void*) GetProcAddress(h, "nvmlInit_v2");
+ const auto bus = (ByBusFn) (void*) GetProcAddress(h, "nvmlDeviceGetHandleByPciBusId_v2");
+ const auto m = (MemFn) (void*) GetProcAddress(h, "nvmlDeviceGetMemoryInfo");
+ if (init == nullptr || bus == nullptr || m == nullptr || init() != 0) return;
+ by_bus = bus;
+ mem = m;
+ }
+ };
+ static const Lib lib;
+ if (lib.mem == nullptr) return false;
+ int dev = 0;
+ cudaDeviceProp prop{};
+ if (cudaGetDevice(&dev) != cudaSuccess || cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;
+ char bus[32];
+ std::snprintf(bus, sizeof bus, "%08X:%02X:%02X.0", (unsigned) prop.pciDomainID, (unsigned) prop.pciBusID,
+ (unsigned) prop.pciDeviceID);
+ void* handle = nullptr;
+ Memory m{};
+ if (lib.by_bus(bus, &handle) != 0 || lib.mem(handle, &m) != 0) return false;
+ out = (size_t) m.free;
+ return true;
+}
+
+} // namespace
+#endif
+
+ namespace {
+ // STRATA_EXPERT_NIBBLE: a write into a sealed cache (see ExpertCache::seal). Says which call and stops: the slots hold
+ // 4-bit codes the GGUF bytes would overwrite, or a GGUF expert where a 4-bit one is expected - a wrong reply, not an error.
+ [[noreturn]] void sealed_write(const char* call, int32_t slot) {
+ std::fprintf(stderr, "strata: STRATA_EXPERT_NIBBLE: ExpertCache::%s(slot %d) after the cache was sealed - a GGUF "
+ "blob would be written into the 4-bit expert slots: stopping\n", call, (int) slot);
+ std::fflush(stderr);
+ std::abort();
+ }
+ } // namespace
+
// See the header. NVIDIA's DGX Spark Porting Guide (section 5.5, "Memory reporting on UMA systems") recommends the same:
// not to rely on cudaMemGetInfo alone but to count the memory the OS can reclaim. Swap is not counted here (unlike
// NVIDIA's reference snippet): an expert cache that pushes the system into swap would be far slower than a smaller one.
diff --cc src/core/verify.cpp
index a87e995d2,727fea0e1..79b3b89b4
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@@ -4,7 -4,7 +4,8 @@@
#include "strata/core/sm120_series.hpp"
#include "strata/kernels/gfx_arch.hpp"
#include "strata/core/verify.hpp"
+#include "strata/core/window_evict.hpp"
+ #include "strata/core/expert_nibble.hpp"
#include "strata/core/pin_budget.hpp"
#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)
#include "strata/core/host_graph_launch.hpp"
@@@ -3714,27 -3748,8 +3767,27 @@@ bool Verifier::evict_batch_graph(const
return true;
}
+bool Verifier::evict_window_graph(int keep_t, bool& evicted, std::string& err) {
+ evicted = false;
+ if (no_window_eviction_ || keep_t < 1) return true;
+ bool have[9] = {};
- for (int t = 1; t <= max_t_ && t < 9; ++t) have[t] = (ar_off_ ? exec_nr_[t] : exec_[t]) != nullptr;
++ for (int t = 1; t <= max_t_ && t < 9; ++t) have[t] = (ar_off_ ? exec_nr_[sel_long_][t] : exec_[sel_long_][t]) != nullptr;
+ const int victim = strata::core::pick_window_victim(have, std::min(max_t_, 8), keep_t, prelaunched_);
+ if (victim == 0) return true;
+ if (cudaStreamSynchronize(cs_) != cudaSuccess) {
+ err = "verify: synchronizing before window graph eviction failed";
+ return false;
+ }
- cudaGraphExec_t& e = ar_off_ ? exec_nr_[victim] : exec_[victim];
++ cudaGraphExec_t& e = ar_off_ ? exec_nr_[sel_long_][victim] : exec_[sel_long_][victim];
+ cudaGraphExecDestroy(e);
+ e = nullptr;
- (ar_off_ ? launched_nr_ : launched_)[victim] = false; // the next graph of that size is a cold first launch again
++ (ar_off_ ? launched_nr_ : launched_)[sel_long_][victim] = false; // the next graph of that size is a cold first launch again
+ evicted = true;
+ return true;
+}
+
bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key,
- const char* what, std::string& err) {
+ const char* what, std::string& err, int keep_t) {
// #997: every layout of active slots keeps its own graph pair (20-30 MiB each on an L40S), and with an expert cache
// sized to the reserve the next window graph - a batch layout or a one-request window captured on first use - can
// find no VRAM left: free the least recently used layouts until it fits (they are captured again when needed)
@@@ -4256,13 -4257,14 +4309,19 @@@ bool Verifier::capture_all(std::string
if (g_ == nullptr) { err = "verify: capture_all before init"; return false; }
if (remote_opt_ != nullptr) { err = "verify: pipelined windows do not serve --remote-expert-opt"; return false; }
if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }
+ struct NoEvict { // every size is wanted at once here: a capture must not free another size's graph
+ bool& f;
+ explicit NoEvict(bool& flag) : f(flag) { f = true; }
+ ~NoEvict() { f = false; }
+ } no_evict(no_window_eviction_);
- for (int T = 1; T <= max_t_; ++T)
- if (!capture(T, err)) return false;
+ // STRATA_SELECT_LONG: both select variants (see select_long_at): pl_launch takes the one its window's context
+ // needs, and nothing can be captured once pipelined windows are in flight. Without it only [0]
+ for (int sl = 0; sl < (select_long_on() ? 2 : 1); ++sl) {
+ sel_long_ = sl;
+ for (int T = 1; T <= max_t_; ++T)
+ if (!capture(T, err)) return false;
+ }
+ sel_long_ = 0;
if (!capture_commit(err)) return false;
if ((ev_done_ == nullptr && cudaEventCreateWithFlags(&ev_done_, cudaEventDisableTiming) != cudaSuccess) ||
(ev_commit_ == nullptr && cudaEventCreateWithFlags(&ev_commit_, cudaEventDisableTiming) != cudaSuccess)) {
diff --cc src/prefill/prefill.cpp
index 8b08b511c,2e628967a..68b556466
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@@ -4792,9 -4855,16 +4933,16 @@@ bool Prefill::run_impl(const int64_t* t
const auto& f = lay.fmt[(size_t) l];
if (gg_nslots > 0) { // the copies land in order: the last one covers the others
pt.mark(kPfWaitCopy, cs);
- cudaStreamWaitEvent(m.cs, m.copied[gg_slots[gg_nslots - 1]], 0);
+ wait_copied(gg_ents[gg_nslots - 1], (size_t) gg_slots[gg_nslots - 1]);
pt.mark(kPfDequant, cs);
}
+ #if STRATA_EXPERT_NIBBLE_BUILT
+ if (nib_l.on) { // the GGUF blocks rebuilt from the 4-bit slots: the bytes the copy gives
+ if (const int rc = core::nibble::gather_gguf(nib_l, gg.blob, gg.first, gg.n, m.grp_gu, mmq_gub,
+ m.grp_d, mmq_db, zero, m.cs); rc != 0)
+ core::nibble::fatal(l, "the prompt path's group gather", cudaGetErrorString((cudaError_t) rc));
+ } else
+ #endif
// NVFP4: each expert's scales go next to its group slot in the same launch
if (!mmq::gather_native_group(gg, f.up_off, mmq_gub / 2, f.down_off, mmq_db, m.grp_gu, mmq_gub,
m.grp_d, mmq_db, m.cs, f.tail_off,
diff --cc src/program/generate.cpp
index 54d221cd4,d0de8560c..50692887c
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@@ -155,52 -156,8 +156,53 @@@ namespace strata::metal_engine { void s
#include <set>
#include <vector>
+// logprobs=K / allowed=...: one `LP` line for the token just emitted, from `ver`'s head
+// logits for the window row it came from (before sampling, penalties and temperature).
+// Format matches the SYCL port's LP line so one server-side reader serves both.
+// Called only when the request asked (`req_logprobs >= 0`): a request without logprobs never reaches it, so the
+// default path does no work for this and prints exactly what it printed before.
+static void emit_lp_line(const strata::core::Verifier& ver, int t, int32_t token, int k,
+ const std::vector<int32_t>& allowed) {
+ if (k < 0) return;
+ static std::vector<float> row;
+ row.resize((size_t) ver.vocab());
+ if (!ver.copy_logits(t, row.data()) || token < 0 || (size_t) token >= row.size()) {
+ std::printf("LP nan\n");
+ return;
+ }
+ float mx = -INFINITY;
+ for (const float v : row) mx = std::max(mx, v);
+ double se = 0.0;
+ for (const float v : row) se += std::exp((double) v - (double) mx);
+ const double lse = (double) mx + std::log(se);
+ std::printf("LP %.6f", (double) row[(size_t) token] - lse);
+ if (!allowed.empty()) {
+ // Only the ids the caller named; the server renormalises over them, which is what allowed_token_ids means.
+ // The server refuses an id past the vocabulary with a 400 only when it can read the vocabulary size from the
+ // tokenizer. The engine's own vocab() is the authority: an id past it is left off the line, so the logits
+ // are never read out of bounds.
+ for (const int32_t id : allowed) {
+ if (id == token || id < 0 || (int64_t) id >= ver.vocab()) continue;
+ std::printf(" %d:%.6f", (int) id, (double) row[(size_t) id] - lse);
+ }
+ std::printf("\n");
+ return;
+ }
+ if (k > 0) {
+ static std::vector<int32_t> ord;
+ ord.resize(row.size());
+ for (size_t v = 0; v < row.size(); ++v) ord[v] = (int32_t) v;
+ const int kk = std::min<int>(k, (int) row.size());
+ std::partial_sort(ord.begin(), ord.begin() + kk, ord.end(),
+ [&](int32_t a, int32_t b) { return row[(size_t) a] > row[(size_t) b]; });
+ for (int j = 0; j < kk; ++j)
+ std::printf(" %d:%.6f", ord[(size_t) j], (double) row[(size_t) ord[(size_t) j]] - lse);
+ }
+ std::printf("\n");
+}
+
namespace {
+ namespace nib = strata::core::nibble;
// Windows' WDDM driver model: native Windows, or WSL2 (its GPU goes through /dev/dxg to the Windows driver). There,
// pinning a large arena into two CUDA contexts leaves WDDM refusing every later allocation (the 5080 + 3090 rig);
// a Linux driver has no such limit (#253: the 8 GiB cap cost a 4090 + 3060 split 3x of its prompt speed).