Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge f-027-gh-eng-k1 (#3867 #3751 #3453 #3943 #3820 #3752 #3455: Pascal and sm_89 opt-in kernel ports)
Niko1221551f1cdc301a
diff --cc src/core/expert_cache.cppindex f499378c3,72523b9cd..d670c8da7--- a/src/core/expert_cache.cpp+++ b/src/core/expert_cache.cpp@@@ -22,65 -22,19 +22,76 @@@#include <utility>#include <cstring>+// (not in a SYCL build: its CUDA names are a compatibility layer without the PCI ids NVML is matched by)+#if defined(_WIN32) && !defined(STRATA_EC_NO_VMM) && !defined(SYCL_LANGUAGE_VERSION)+#ifndef WIN32_LEAN_AND_MEAN+#define WIN32_LEAN_AND_MEAN+#endif+#ifndef NOMINMAX+#define NOMINMAX+#endif+#include <windows.h>+#define STRATA_EC_NVML_FREE 1+#endif+namespace strata::core {+#if defined(STRATA_EC_NVML_FREE)+namespace {++// STRATA_VRAM_FREE=device (opt-in, Windows + NVIDIA). Under WDDM cudaMemGetInfo answers for THIS process: memory+// another process holds on the card (the image encoder, a browser, another engine) is not in its free figure.+// Measured on an RTX 5090, driver 610.88: with 6 GiB written by another process, cudaMemGetInfo in a second process+// still read 30,994 MiB free where NVML read 24,348. NVML (nvml.dll, installed with every driver) counts the whole+// card. The library is loaded from the system directory only, at first use; false when it cannot answer.+bool nvml_device_free(size_t& out) {+ struct Memory { unsigned long long total, free, used; };+ typedef int (*InitFn)(void);+ typedef int (*ByBusFn)(const char*, void**);+ typedef int (*MemFn)(void*, Memory*);+ struct Lib {+ ByBusFn by_bus = nullptr;+ MemFn mem = nullptr;+ Lib() {+ HMODULE h = LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32);+ if (h == nullptr) return;+ const auto init = (InitFn) (void*) GetProcAddress(h, "nvmlInit_v2");+ const auto bus = (ByBusFn) (void*) GetProcAddress(h, "nvmlDeviceGetHandleByPciBusId_v2");+ const auto m = (MemFn) (void*) GetProcAddress(h, "nvmlDeviceGetMemoryInfo");+ if (init == nullptr || bus == nullptr || m == nullptr || init() != 0) return;+ by_bus = bus;+ mem = m;+ }+ };+ static const Lib lib;+ if (lib.mem == nullptr) return false;+ int dev = 0;+ cudaDeviceProp prop{};+ if (cudaGetDevice(&dev) != cudaSuccess || cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;+ char bus[32];+ std::snprintf(bus, sizeof bus, "%08X:%02X:%02X.0", (unsigned) prop.pciDomainID, (unsigned) prop.pciBusID,+ (unsigned) prop.pciDeviceID);+ void* handle = nullptr;+ Memory m{};+ if (lib.by_bus(bus, &handle) != 0 || lib.mem(handle, &m) != 0) return false;+ out = (size_t) m.free;+ return true;+}++} // namespace+#endif++ namespace {+ // STRATA_EXPERT_NIBBLE: a write into a sealed cache (see ExpertCache::seal). Says which call and stops: the slots hold+ // 4-bit codes the GGUF bytes would overwrite, or a GGUF expert where a 4-bit one is expected - a wrong reply, not an error.+ [[noreturn]] void sealed_write(const char* call, int32_t slot) {+ std::fprintf(stderr, "strata: STRATA_EXPERT_NIBBLE: ExpertCache::%s(slot %d) after the cache was sealed - a GGUF "+ "blob would be written into the 4-bit expert slots: stopping\n", call, (int) slot);+ std::fflush(stderr);+ std::abort();+ }+ } // namespace+// See the header. NVIDIA's DGX Spark Porting Guide (section 5.5, "Memory reporting on UMA systems") recommends the same:// not to rely on cudaMemGetInfo alone but to count the memory the OS can reclaim. Swap is not counted here (unlike// NVIDIA's reference snippet): an expert cache that pushes the system into swap would be far slower than a smaller one.diff --cc src/core/verify.cppindex a87e995d2,727fea0e1..79b3b89b4--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@@ -4,7 -4,7 +4,8 @@@#include "strata/core/sm120_series.hpp"#include "strata/kernels/gfx_arch.hpp"#include "strata/core/verify.hpp"+#include "strata/core/window_evict.hpp"+ #include "strata/core/expert_nibble.hpp"#include "strata/core/pin_budget.hpp"#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)#include "strata/core/host_graph_launch.hpp"@@@ -3714,27 -3748,8 +3767,27 @@@ bool Verifier::evict_batch_graph(constreturn true;}+bool Verifier::evict_window_graph(int keep_t, bool& evicted, std::string& err) {+ evicted = false;+ if (no_window_eviction_ || keep_t < 1) return true;+ bool have[9] = {};- for (int t = 1; t <= max_t_ && t < 9; ++t) have[t] = (ar_off_ ? exec_nr_[t] : exec_[t]) != nullptr;++ for (int t = 1; t <= max_t_ && t < 9; ++t) have[t] = (ar_off_ ? exec_nr_[sel_long_][t] : exec_[sel_long_][t]) != nullptr;+ const int victim = strata::core::pick_window_victim(have, std::min(max_t_, 8), keep_t, prelaunched_);+ if (victim == 0) return true;+ if (cudaStreamSynchronize(cs_) != cudaSuccess) {+ err = "verify: synchronizing before window graph eviction failed";+ return false;+ }- cudaGraphExec_t& e = ar_off_ ? exec_nr_[victim] : exec_[victim];++ cudaGraphExec_t& e = ar_off_ ? exec_nr_[sel_long_][victim] : exec_[sel_long_][victim];+ cudaGraphExecDestroy(e);+ e = nullptr;- (ar_off_ ? launched_nr_ : launched_)[victim] = false; // the next graph of that size is a cold first launch again++ (ar_off_ ? launched_nr_ : launched_)[sel_long_][victim] = false; // the next graph of that size is a cold first launch again+ evicted = true;+ return true;+}+bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, const std::vector<int>& key,- const char* what, std::string& err) {+ const char* what, std::string& err, int keep_t) {// #997: every layout of active slots keeps its own graph pair (20-30 MiB each on an L40S), and with an expert cache// sized to the reserve the next window graph - a batch layout or a one-request window captured on first use - can// find no VRAM left: free the least recently used layouts until it fits (they are captured again when needed)@@@ -4256,13 -4257,14 +4309,19 @@@ bool Verifier::capture_all(std::stringif (g_ == nullptr) { err = "verify: capture_all before init"; return false; }if (remote_opt_ != nullptr) { err = "verify: pipelined windows do not serve --remote-expert-opt"; return false; }if (released_.load()) { err = "verify: an earlier window never finished on the GPU (#267); restart the engine"; return false; }+ struct NoEvict { // every size is wanted at once here: a capture must not free another size's graph+ bool& f;+ explicit NoEvict(bool& flag) : f(flag) { f = true; }+ ~NoEvict() { f = false; }+ } no_evict(no_window_eviction_);- for (int T = 1; T <= max_t_; ++T)- if (!capture(T, err)) return false;+ // STRATA_SELECT_LONG: both select variants (see select_long_at): pl_launch takes the one its window's context+ // needs, and nothing can be captured once pipelined windows are in flight. Without it only [0]+ for (int sl = 0; sl < (select_long_on() ? 2 : 1); ++sl) {+ sel_long_ = sl;+ for (int T = 1; T <= max_t_; ++T)+ if (!capture(T, err)) return false;+ }+ sel_long_ = 0;if (!capture_commit(err)) return false;if ((ev_done_ == nullptr && cudaEventCreateWithFlags(&ev_done_, cudaEventDisableTiming) != cudaSuccess) ||(ev_commit_ == nullptr && cudaEventCreateWithFlags(&ev_commit_, cudaEventDisableTiming) != cudaSuccess)) {diff --cc src/prefill/prefill.cppindex 8b08b511c,2e628967a..68b556466--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@@ -4792,9 -4855,16 +4933,16 @@@ bool Prefill::run_impl(const int64_t* tconst auto& f = lay.fmt[(size_t) l];if (gg_nslots > 0) { // the copies land in order: the last one covers the otherspt.mark(kPfWaitCopy, cs);- cudaStreamWaitEvent(m.cs, m.copied[gg_slots[gg_nslots - 1]], 0);+ wait_copied(gg_ents[gg_nslots - 1], (size_t) gg_slots[gg_nslots - 1]);pt.mark(kPfDequant, cs);}+ #if STRATA_EXPERT_NIBBLE_BUILT+ if (nib_l.on) { // the GGUF blocks rebuilt from the 4-bit slots: the bytes the copy gives+ if (const int rc = core::nibble::gather_gguf(nib_l, gg.blob, gg.first, gg.n, m.grp_gu, mmq_gub,+ m.grp_d, mmq_db, zero, m.cs); rc != 0)+ core::nibble::fatal(l, "the prompt path's group gather", cudaGetErrorString((cudaError_t) rc));+ } else+ #endif// NVFP4: each expert's scales go next to its group slot in the same launchif (!mmq::gather_native_group(gg, f.up_off, mmq_gub / 2, f.down_off, mmq_db, m.grp_gu, mmq_gub,m.grp_d, mmq_db, m.cs, f.tail_off,diff --cc src/program/generate.cppindex 54d221cd4,d0de8560c..50692887c--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@@ -155,52 -156,8 +156,53 @@@ namespace strata::metal_engine { void s#include <set>#include <vector>+// logprobs=K / allowed=...: one `LP` line for the token just emitted, from `ver`'s head+// logits for the window row it came from (before sampling, penalties and temperature).+// Format matches the SYCL port's LP line so one server-side reader serves both.+// Called only when the request asked (`req_logprobs >= 0`): a request without logprobs never reaches it, so the+// default path does no work for this and prints exactly what it printed before.+static void emit_lp_line(const strata::core::Verifier& ver, int t, int32_t token, int k,+ const std::vector<int32_t>& allowed) {+ if (k < 0) return;+ static std::vector<float> row;+ row.resize((size_t) ver.vocab());+ if (!ver.copy_logits(t, row.data()) || token < 0 || (size_t) token >= row.size()) {+ std::printf("LP nan\n");+ return;+ }+ float mx = -INFINITY;+ for (const float v : row) mx = std::max(mx, v);+ double se = 0.0;+ for (const float v : row) se += std::exp((double) v - (double) mx);+ const double lse = (double) mx + std::log(se);+ std::printf("LP %.6f", (double) row[(size_t) token] - lse);+ if (!allowed.empty()) {+ // Only the ids the caller named; the server renormalises over them, which is what allowed_token_ids means.+ // The server refuses an id past the vocabulary with a 400 only when it can read the vocabulary size from the+ // tokenizer. The engine's own vocab() is the authority: an id past it is left off the line, so the logits+ // are never read out of bounds.+ for (const int32_t id : allowed) {+ if (id == token || id < 0 || (int64_t) id >= ver.vocab()) continue;+ std::printf(" %d:%.6f", (int) id, (double) row[(size_t) id] - lse);+ }+ std::printf("\n");+ return;+ }+ if (k > 0) {+ static std::vector<int32_t> ord;+ ord.resize(row.size());+ for (size_t v = 0; v < row.size(); ++v) ord[v] = (int32_t) v;+ const int kk = std::min<int>(k, (int) row.size());+ std::partial_sort(ord.begin(), ord.begin() + kk, ord.end(),+ [&](int32_t a, int32_t b) { return row[(size_t) a] > row[(size_t) b]; });+ for (int j = 0; j < kk; ++j)+ std::printf(" %d:%.6f", ord[(size_t) j], (double) row[(size_t) ord[(size_t) j]] - lse);+ }+ std::printf("\n");+}+namespace {+ namespace nib = strata::core::nibble;// Windows' WDDM driver model: native Windows, or WSL2 (its GPU goes through /dev/dxg to the Windows driver). There,// pinning a large arena into two CUDA contexts leaves WDDM refusing every later allocation (the 5080 + 3090 rig);// a Linux driver has no such limit (#253: the 8 GiB cap cost a 4090 + 3060 split 3x of its prompt speed).