Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Later-card VRAM: audit fixes for failed allocations, the reserve arithmetic as a tested helper, a per-card out-of-memory report (#3638 #3702 #3797)
Niko12212a9a67c174e5
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 09459ac06..4a14a173b 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -315,6 +315,18 @@ if(STRATA_BUILD_TESTS)target_include_directories(split_rules_test PRIVATE include)add_test(NAME split_rules_test COMMAND split_rules_test)+ # #3797: the out-of-memory report at start (per card free/total, the largest other programs): the text is a pure+ # function; the driver query (NVML, loaded at run time) is only checked to be safe to call. No GPU needed.+ add_executable(gpu_report_test tests/program/gpu_report_test.cpp)+ target_include_directories(gpu_report_test PRIVATE include)+ target_link_libraries(gpu_report_test PRIVATE ${CMAKE_DL_LIBS})+ add_test(NAME gpu_report_test COMMAND gpu_report_test)++ # #3702: the later-stage reserve of the layer split search (header-only arithmetic, pinned against the old expression)+ add_executable(stage_reserve_test tests/program/stage_reserve_test.cpp)+ target_include_directories(stage_reserve_test PRIVATE include)+ add_test(NAME stage_reserve_test COMMAND stage_reserve_test)+# the prompt planner's host-memory model (#2824): header-only, no GPUadd_executable(prefill_plan_test tests/program/prefill_plan_test.cpp)target_include_directories(prefill_plan_test PRIVATE include)diff --git a/include/strata/program/gpu_report.hpp b/include/strata/program/gpu_report.hppnew file mode 100644index 000000000..efc6f6567--- /dev/null+++ b/include/strata/program/gpu_report.hpp@@ -0,0 +1,217 @@+// include/strata/program/gpu_report.hpp - #3797: what to tell the user when a GPU allocation at start fails.+//+// "cudaMalloc(818947840) for the weight arena failed: 320 MiB of 21998 MiB free" says that the card is full, not by+// whom. The usual answer is another program (or an engine that is still exiting), so the report names, per card, the+// memory that is free and the largest other processes that hold some. The text is a warning added to the message that+// was already there; nothing about the failure changes.+//+// `format_gpu_memory_report` is a pure function (tests/program/gpu_report_test.cpp). `query_other_processes` asks+// the NVIDIA driver library (NVML, loaded at run time: no link-time dependency, so a build without it, an AMD card,+// a missing driver library or a failed call just returns false and the report leaves the process list out). It+// never throws and never ends the process.+#pragma once++#include <algorithm>+#include <cstdint>+#include <cstdio>+#include <string>+#include <vector>++#if defined(_WIN32)+#ifndef WIN32_LEAN_AND_MEAN+#define WIN32_LEAN_AND_MEAN+#endif+#ifndef NOMINMAX+#define NOMINMAX+#endif+#include <windows.h>+#else+#include <dlfcn.h>+#endif++namespace strata::program::gpu_report {++struct Process {+ uint32_t pid = 0;+ std::string name; ///< the program's file name; empty when the system would not say+ uint64_t used_mib = 0; ///< 0 = the driver does not report it (Windows WDDM does not for most programs)+};++struct Card {+ int index = 0; ///< the CUDA device number+ std::string name; ///< "NVIDIA GeForce RTX 2080 Ti"; may be empty+ uint64_t free_mib = 0;+ uint64_t total_mib = 0;+ bool processes_known = false; ///< the driver answered the process query (the list may still be empty)+ std::vector<Process> processes; ///< every process that holds memory on this card, in any order+};++namespace detail {+inline std::string size_text(uint64_t mib) {+ char b[48];+ if (mib >= 10240) std::snprintf(b, sizeof b, "%.1f GiB", (double) mib / 1024.0);+ else std::snprintf(b, sizeof b, "%llu MiB", (unsigned long long) mib);+ return b;+}+inline std::string base_name(const std::string& path) {+ const size_t cut = path.find_last_of("/\\");+ return cut == std::string::npos ? path : path.substr(cut + 1);+}+} // namespace detail++/// One line per card: the free and total memory and, when the driver listed them, the largest other processes+/// (`self_pid` and any process of 0 MiB that the driver does list are left out of "largest"; at most `max_procs`+/// are named). Ends with no newline. `cards` empty: an empty string.+inline std::string format_gpu_memory_report(const std::vector<Card>& cards, uint32_t self_pid, size_t max_procs = 3) {+ std::string out;+ for (const Card& c : cards) {+ if (!out.empty()) out += "\n";+ out += "strata generate: CUDA" + std::to_string(c.index);+ if (!c.name.empty()) out += " (" + c.name + ")";+ out += ": " + std::to_string((unsigned long long) c.free_mib) + " MiB free of " ++ std::to_string((unsigned long long) c.total_mib) + " MiB"; // (as the engine's other memory lines)+ if (!c.processes_known) {+ out += "; the driver does not say which programs use it here";+ continue;+ }+ std::vector<Process> others;+ size_t unsized = 0;+ for (const Process& p : c.processes) {+ if (p.pid == self_pid) continue;+ if (p.used_mib == 0) { ++unsized; continue; }+ others.push_back(p);+ }+ std::sort(others.begin(), others.end(), [](const Process& a, const Process& b) {+ return a.used_mib != b.used_mib ? a.used_mib > b.used_mib : a.pid < b.pid;+ });+ if (others.empty() && unsized == 0) {+ out += "; the driver lists no other program using it";+ continue;+ }+ out += "; other programs on it: ";+ for (size_t i = 0; i < others.size() && i < max_procs; ++i) {+ if (i > 0) out += ", ";+ const std::string n = detail::base_name(others[i].name);+ out += (n.empty() ? "pid " + std::to_string(others[i].pid) : n + " (pid " + std::to_string(others[i].pid) + ")") + " " ++ detail::size_text(others[i].used_mib);+ }+ if (others.size() > max_procs) out += ", and " + std::to_string(others.size() - max_procs) + " more";+ if (unsized > 0) out += std::string(others.empty() ? "" : "; ") + std::to_string(unsized) ++ (unsized == 1 ? " program whose memory the driver does not report" : " programs whose memory the driver does not report");+ }+ return out;+}++/// "The split planned to keep N MiB free on this card, M MiB are free" for the low-VRAM warning: how far the engine's+/// own estimate was off. Empty when the card has at least what was planned (nothing to explain).+inline std::string planned_vs_free_text(int64_t planned_mib, int64_t free_mib) {+ if (planned_mib <= 0 || free_mib >= planned_mib) return {};+ return "the layer split planned to keep " + std::to_string((long long) planned_mib) + " MiB free here after loading; " ++ std::to_string((long long) std::max<int64_t>(free_mib, 0)) + " MiB are free, so " ++ std::to_string((long long) (planned_mib - std::max<int64_t>(free_mib, 0))) + " MiB went to something the plan did not count";+}++// ---------------------------------------------------------------------------------------------- the driver query+namespace detail {+struct NvmlProc { // nvmlProcessInfo_v2_t / the layout the _v3 calls fill+ unsigned int pid;+ unsigned long long usedGpuMemory;+ unsigned int gpuInstanceId;+ unsigned int computeInstanceId;+};+constexpr unsigned long long kNvmlNotAvailable = ~0ull;+using NvmlInit = int (*)();+using NvmlHandleByPci = int (*)(const char*, void**);+using NvmlProcs = int (*)(void*, unsigned int*, NvmlProc*);+using NvmlProcName = int (*)(unsigned int, char*, unsigned int);++struct Nvml {+ void* lib = nullptr;+ NvmlHandleByPci by_pci = nullptr;+ NvmlProcs compute = nullptr;+ NvmlProcs graphics = nullptr;+ NvmlProcName name = nullptr;+ NvmlInit shutdown = nullptr;+};++inline void* open_lib() {+#if defined(_WIN32)+ return (void*) LoadLibraryA("nvml.dll");+#else+ for (const char* n : {"libnvidia-ml.so.1", "libnvidia-ml.so"})+ if (void* h = dlopen(n, RTLD_LAZY | RTLD_LOCAL)) return h;+ return nullptr;+#endif+}+inline void* sym(void* lib, const char* n) {+#if defined(_WIN32)+ return (void*) GetProcAddress((HMODULE) lib, n);+#else+ return dlsym(lib, n);+#endif+}+inline void close_lib(void* lib) {+#if defined(_WIN32)+ if (lib) FreeLibrary((HMODULE) lib);+#else+ if (lib) dlclose(lib);+#endif+}+} // namespace detail++/// The processes that hold memory on the card with this PCI bus id ("0000:65:00.0", what cudaDeviceGetPCIBusId+/// returns). true: the driver answered (the list may be empty). false: it could not be asked. Best effort.+inline bool query_other_processes(const char* pci_bus_id, std::vector<Process>& out) {+ out.clear();+ if (pci_bus_id == nullptr || pci_bus_id[0] == '\0') return false;+ detail::Nvml n;+ n.lib = detail::open_lib();+ if (n.lib == nullptr) return false;+ bool ok = false;+ do {+ auto init = (detail::NvmlInit) detail::sym(n.lib, "nvmlInit_v2");+ n.shutdown = (detail::NvmlInit) detail::sym(n.lib, "nvmlShutdown");+ n.by_pci = (detail::NvmlHandleByPci) detail::sym(n.lib, "nvmlDeviceGetHandleByPciBusId_v2");+ n.name = (detail::NvmlProcName) detail::sym(n.lib, "nvmlSystemGetProcessName");+ // _v3 and _v2 take the 24-byte record above; the unversioned (v1) call has another record, so it is not used+ n.compute = (detail::NvmlProcs) detail::sym(n.lib, "nvmlDeviceGetComputeRunningProcesses_v3");+ if (!n.compute) n.compute = (detail::NvmlProcs) detail::sym(n.lib, "nvmlDeviceGetComputeRunningProcesses_v2");+ n.graphics = (detail::NvmlProcs) detail::sym(n.lib, "nvmlDeviceGetGraphicsRunningProcesses_v3");+ if (!n.graphics) n.graphics = (detail::NvmlProcs) detail::sym(n.lib, "nvmlDeviceGetGraphicsRunningProcesses_v2");+ if (!init || !n.by_pci || !n.compute || init() != 0) break;+ void* dev = nullptr;+ if (n.by_pci(pci_bus_id, &dev) != 0 || dev == nullptr) { if (n.shutdown) n.shutdown(); break; }+ for (detail::NvmlProcs call : {n.compute, n.graphics}) {+ if (!call) continue;+ std::vector<detail::NvmlProc> recs(128);+ unsigned int count = (unsigned int) recs.size();+ int rc = call(dev, &count, recs.data());+ if (rc == 7 && count > recs.size()) { // NVML_ERROR_INSUFFICIENT_SIZE: `count` is the number+ recs.resize((size_t) count + 8);+ count = (unsigned int) recs.size();+ rc = call(dev, &count, recs.data());+ }+ if (rc != 0) continue;+ ok = true;+ for (unsigned int i = 0; i < count && i < recs.size(); ++i) {+ bool seen = false;+ for (Process& p : out)+ if (p.pid == recs[i].pid) { seen = true; break; }+ if (seen) continue;+ Process p;+ p.pid = recs[i].pid;+ p.used_mib = recs[i].usedGpuMemory == detail::kNvmlNotAvailable ? 0 : (uint64_t) (recs[i].usedGpuMemory >> 20);+ if (n.name) {+ char buf[260] = {};+ if (n.name(p.pid, buf, (unsigned int) sizeof buf) == 0) p.name = buf;+ }+ out.push_back(std::move(p));+ }+ }+ if (n.shutdown) n.shutdown();+ } while (false);+ detail::close_lib(n.lib);+ return ok;+}++} // namespace strata::program::gpu_reportdiff --git a/include/strata/program/stage_reserve.hpp b/include/strata/program/stage_reserve.hppnew file mode 100644index 000000000..f76435303--- /dev/null+++ b/include/strata/program/stage_reserve.hpp@@ -0,0 +1,61 @@+// include/strata/program/stage_reserve.hpp - what the layer split keeps free on a card when it sizes the expert cache.+//+// This is the arithmetic of `stage_room` in src/program/generate.cpp, moved here unchanged so a host test can pin it+// (tests/program/stage_reserve_test.cpp). `reserve_mib` is exactly the sum that function subtracted from the card's+// free memory before; the default placement does not change.+//+// #3702: on an unequal pair (RTX 3060 + RTX 5060 Ti) the last card was left with 68 MiB after loading although+// 700 + 96 MiB were reserved for it, and the verify windows' graphs (captured on first use, 4-6 MiB each and once 42 MiB)+// then ran it out of memory. `proposed_window_mib` below is a PROPOSAL for a larger window term; nothing calls it+// (it needs the model gate in phaseA-tests/gh027/gates/3702-later-reserve.sh before it can become the default).+#pragma once++#include <algorithm>+#include <cstdint>++namespace strata::program::stage_reserve {++constexpr int64_t kWindowMib = 96; ///< the verify windows on a later stage; 75 MiB measured, rounded up+constexpr int64_t kDrafterMib = 1000; ///< the MTP drafter (839 MiB) + the head, on the last stage only++struct Inputs {+ int64_t base_reserve_mib = 0; ///< --vram-reserve-later-mib on a later stage when given, else --vram-reserve-mib+ int64_t prompt_buffers_mib = 0; ///< the prompt path's own buffers (0 when they are borrowed from the cache)+ bool later = false; ///< a stage after the first+ bool drafter = false; ///< the stage that carries the MTP drafter and the head+ int64_t pipe_window_mib = 0; ///< --pipeline-windows: a second verifier on every stage+ int64_t pipe_snap_mib = 0; ///< --pipeline-windows 2 or more: the first stage's second GDN state+};++/// What the plan keeps free on the card, in MiB (generate.cpp's `reserve` before the shift).+inline int64_t reserve_mib(const Inputs& in) {+ return in.base_reserve_mib + in.prompt_buffers_mib + (in.later ? kWindowMib : in.pipe_snap_mib) + in.pipe_window_mib ++ (in.drafter ? kDrafterMib : 0);+}++/// The room the cache may use: the free bytes less the reserve, never below 0.+inline int64_t room_bytes(int64_t free_bytes, const Inputs& in) {+ return std::max<int64_t>(free_bytes - (reserve_mib(in) << 20), 0);+}++// ---------------------------------------------------------------------------------------------------- the proposal+constexpr int64_t kGraphMib = 6; ///< one window graph, captured on first use (log of #3702: 4, 6, 6)+constexpr int64_t kInstantiateBurstMib = 48; ///< the largest single step seen (42 MiB in #3702): room kept for one more++/// PROPOSAL, unused: the window term of a later stage when the windows of every size 1..max_window_t can still be+/// captured after loading: the first window's arena (kWindowMib already holds it), a graph for each further size, and+/// one burst.+inline int64_t proposed_window_mib(int max_window_t) {+ const int64_t sizes = std::max<int64_t>(max_window_t, 1);+ return kWindowMib + (sizes - 1) * kGraphMib + kInstantiateBurstMib;+}++/// PROPOSAL, unused: reserve_mib with the proposed window term on a later stage.+inline int64_t proposed_reserve_mib(const Inputs& in, int max_window_t) {+ Inputs p = in;+ int64_t r = reserve_mib(p);+ if (in.later) r += proposed_window_mib(max_window_t) - kWindowMib;+ return r;+}++} // namespace strata::program::stage_reservediff --git a/src/core/mtp.cpp b/src/core/mtp.cppindex 292a5d2b4..972006137 100644--- a/src/core/mtp.cpp+++ b/src/core/mtp.cpp@@ -1238,16 +1238,19 @@ bool finish_capture(cudaStream_t cs, bool ok, cudaGraphExec_t& exec, const char*if (graph) cudaGraphDestroy(graph);return false;}- if (ce != cudaSuccess || cudaGraphInstantiate(&exec, graph, 0) != cudaSuccess) {+ cudaGraphExec_t got = nullptr; // (a failed instantiate leaves nothing in the member)+ const cudaError_t ie = ce != cudaSuccess ? ce : cudaGraphInstantiate(&got, graph, 0);+ if (ie != cudaSuccess) {if (graph) cudaGraphDestroy(graph);- err = std::string("mtp: ") + what + " capture: " + cudaGetErrorString(ce);+ err = std::string("mtp: ") + what + " capture: " + cudaGetErrorString(ie); // the status that failedreturn false;}+ exec = got;mtp_list_nodes(graph, what);cudaGraphDestroy(graph);- // an explicit upload: the first launch's implicit one blocked behind a device-side spin (verify.cpp)- cudaGraphUpload(exec, cs);- cudaStreamSynchronize(cs);+ // an explicit upload: the first launch's implicit one blocked behind a device-side spin (verify.cpp); only a warm-up,+ // so a failure (no VRAM left) is cleared and the first launch uploads it+ if (cudaGraphUpload(exec, cs) != cudaSuccess || cudaStreamSynchronize(cs) != cudaSuccess) (void) cudaGetLastError();return true;}} // namespacediff --git a/src/core/verify.cpp b/src/core/verify.cppindex 5e97b8f52..193b8f5ba 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -2583,6 +2583,12 @@ bool Verifier::capture(int T, std::string& err) {const cudaError_t us = cudaStreamSynchronize(cs_);std::fprintf(stderr, "strata verify: captured the %d-token window (upload %s, sync %s)\n", T,cudaGetErrorString(ue), cudaGetErrorString(us));+ // The upload only warms the graph; the first launch does it otherwise. A failed one (out of memory on a full card)+ // must not stay pending: the next kernel wrapper's cudaGetLastError would read it and end the engine (#3702).+ if (ue != cudaSuccess || us != cudaSuccess) {+ (void) cudaGetLastError();+ std::fprintf(stderr, "strata verify: the %d-token window's upload was skipped (the first launch does it)\n", T);+ }return true;}@@ -2706,13 +2712,16 @@ bool Verifier::capture_commit(std::string& err) {if (graph) cudaGraphDestroy(graph);return false;}- if (ce != cudaSuccess || cudaGraphInstantiate(&commit_exec_, graph, 0) != cudaSuccess) {+ if (ce != cudaSuccess) {if (graph) cudaGraphDestroy(graph);err = std::string("verify: commit capture: ") + cudaGetErrorString(ce);return false;}+ // (through the same path as the window graphs: on a full card it frees older batch layouts and retries, and the error+ // it reports is the instantiate's own)+ const bool made = instantiate_evicting(commit_exec_, graph, {}, "verify: commit instantiate: ", err, 0);cudaGraphDestroy(graph);- return true;+ return made;}void Verifier::stage_inputs(int T, const int32_t* tokens, int64_t pos0) {@@ -3545,8 +3554,8 @@ bool Verifier::capture_batch(const int* rows, int S, int hbase, std::string& errconst bool made = instantiate_evicting(ex, graph, bkey(rows, S, hbase), "verify: batch instantiate: ", err);cudaGraphDestroy(graph);if (!made) return false;- cudaGraphUpload(ex, cs_);- cudaStreamSynchronize(cs_);+ if (cudaGraphUpload(ex, cs_) != cudaSuccess || cudaStreamSynchronize(cs_) != cudaSuccess)+ (void) cudaGetLastError(); // only a warm-up: the first launch uploads it (see capture)std::string list;for (int t = 0; t < S; ++t) list += (t ? "," : "") + std::to_string(rows[t]);std::fprintf(stderr, "strata verify: captured the batch window over slots %s\n", list.c_str());@@ -3720,14 +3729,17 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, cons// #997: every layout of active slots keeps its own graph pair (20-30 MiB each on an L40S), and with an expert cache// sized to the reserve the next window graph - a batch layout or a one-request window captured on first use - can// find no VRAM left: free the least recently used layouts until it fits (they are captured again when needed)- cudaError_t ie = cudaGraphInstantiate(&ex, graph, 0);+ // (into a local: what a failed instantiate leaves in the output is not specified, and `ex` is a member that+ // capture() reads as "captured" when it is not null)+ cudaGraphExec_t got = nullptr;+ cudaError_t ie = cudaGraphInstantiate(&got, graph, 0);int freed = 0;for (bool evicted = true; ie == cudaErrorMemoryAllocation && evicted;) {(void) cudaGetLastError();if (!evict_batch_graph(key, evicted, err)) return false;if (evicted) {++freed;- ie = cudaGraphInstantiate(&ex, graph, 0);+ ie = cudaGraphInstantiate(&got, graph, 0);}}if (freed > 0)@@ -3740,7 +3752,7 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, consif (!evict_window_graph(keep_t, evicted, err)) return false;if (evicted) {++freed_sizes;- ie = cudaGraphInstantiate(&ex, graph, 0);+ ie = cudaGraphInstantiate(&got, graph, 0);}}if (freed_sizes > 0)@@ -3748,9 +3760,11 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, cons"sizes (they are captured again when used; a larger --vram-reserve-mib or ""--vram-reserve-later-mib keeps them)\n", keep_t, freed_sizes);if (ie != cudaSuccess) {+ got = nullptr;err = std::string(what) + cudaGetErrorString(ie);return false;}+ ex = got;return true;}diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 08b529e6f..e253806f7 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -1600,6 +1600,9 @@ bool Prefill::carve(size_t T, void* alloc) {const uint64_t region = std::max({gdn_set_bytes(T), qsa_set_bytes(T, m.cap, m.max_blocks, m.sel_batch,m.attn_batch, s), moe_set_bytes(T, m.g->n_expert, fz)});uint8_t* base = o.take<uint8_t>((size_t) region, ok);+ // No region: the sub-allocators below would each cudaMalloc their share (hundreds of MiB) on a card that has just+ // shown it has no room, and leave those buffers in `owned`. The failure is reported by the caller.+ if (base == nullptr) return false;m.region = base;m.region_bytes = region;Alloc a;diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 75d404ac9..1adb7e183 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -84,6 +84,8 @@#include "strata/program/logits_selection.hpp"#include "strata/program/conv_cache.hpp"#include "strata/program/split_rules.hpp"+#include "strata/program/stage_reserve.hpp"+#include "strata/program/gpu_report.hpp"#include "strata/prefill/share_rules.hpp"#include "strata/platform/host_limits.hpp"#include "strata/program/split_ring.hpp"@@ -1593,6 +1595,49 @@ int64_t res_check(const int32_t* d_res, const std::vector<int32_t>& host_res, streturn bad;}+/// #3797: when the engine cannot start for lack of GPU memory, the cards' free and total memory and the largest other+/// programs that hold some, as a warning added to the message that says it (program/gpu_report.hpp). Best effort: a+/// card that cannot be asked, or a driver that does not list its programs, is left out or said so; it never changes the+/// failure and never throws. `devices`: the CUDA ordinals to report (duplicates are skipped).+static void print_startup_gpu_report(const std::vector<int>& devices) {+ namespace gr = strata::program::gpu_report;+ std::vector<gr::Card> cards;+ int count = 0, prev = 0;+ if (cudaGetDeviceCount(&count) != cudaSuccess) { (void) cudaGetLastError(); return; }+ if (cudaGetDevice(&prev) != cudaSuccess) { (void) cudaGetLastError(); prev = 0; }+ std::vector<int> seen;+ for (const int d : devices) {+ if (d < 0 || d >= count || std::find(seen.begin(), seen.end(), d) != seen.end()) continue;+ seen.push_back(d);+ if (cudaSetDevice(d) != cudaSuccess) { (void) cudaGetLastError(); continue; } // no context fits there either+ size_t free_b = 0, total_b = 0;+ if (cudaMemGetInfo(&free_b, &total_b) != cudaSuccess) { (void) cudaGetLastError(); continue; }+ gr::Card c;+ c.index = d;+ c.free_mib = (uint64_t) (free_b >> 20);+ c.total_mib = (uint64_t) (total_b >> 20);+ cudaDeviceProp prop{};+ if (cudaGetDeviceProperties(&prop, d) == cudaSuccess) c.name = prop.name;+ else (void) cudaGetLastError();+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)+ char bus[64] = {};+ if (cudaDeviceGetPCIBusId(bus, (int) sizeof bus, d) == cudaSuccess) c.processes_known = gr::query_other_processes(bus, c.processes);+ else (void) cudaGetLastError();+#endif+ cards.push_back(std::move(c));+ }+ if (prev >= 0 && prev < count) (void) cudaSetDevice(prev);+ (void) cudaGetLastError();+ if (cards.empty()) return;+#if defined(_WIN32)+ const uint32_t self = (uint32_t) GetCurrentProcessId();+#else+ const uint32_t self = (uint32_t) getpid();+#endif+ std::fprintf(stderr, "strata generate: GPU memory right now (best effort):\n%s\n",+ gr::format_gpu_memory_report(cards, self).c_str());+}+/// The pre-launch hook's patch: the changed entries of the host's residency table (`idx`) as (index, entry) pairs into/// the mapped `pairs` (room for `cap`); their count, or -1 when they do not fit (then the whole table goes up).int64_t res_patch_pairs(const std::vector<int32_t>& idx, const std::vector<int32_t>& host_res, int32_t* pairs, size_t cap) {@@ -3920,6 +3965,13 @@ int main(int argc, char** argv) {"nvidia-smi / rocm-smi lists them\n",(unsigned long long) pool_bytes, cudaGetErrorString(ce), (unsigned long long) (free_b >> 20),(unsigned long long) (total_b >> 20));+ { // #3797: every card of the run, and who else is on it+ std::vector<int> report_devs;+ int cur = 0;+ if (cudaGetDevice(&cur) == cudaSuccess) report_devs.push_back(cur); else (void) cudaGetLastError();+ for (const int d : split_devs) report_devs.push_back(d);+ print_startup_gpu_report(report_devs);+ }return 1;}mem_mark("the CUDA context and the weight arena");@@ -4417,6 +4469,13 @@ int main(int argc, char** argv) {err.empty() ? "the weight arena does not fit" : err.c_str(),(unsigned long long) (pool_s >> 20), (unsigned long long) (free_b >> 20),(unsigned long long) (total_b >> 20));+ { // #3797: every card of the run, and who else is on it+ std::vector<int> report_devs;+ int cur = 0;+ if (cudaGetDevice(&cur) == cudaSuccess) report_devs.push_back(cur); else (void) cudaGetLastError();+ for (const int d : split_devs) report_devs.push_back(d);+ print_startup_gpu_report(report_devs);+ }return false;}if (!o.native_dense_gguf.empty() && !st.dense.load(o.native_dense_gguf, st.wt, err, o.native_ple_key)) {@@ -4592,8 +4651,8 @@ int main(int argc, char** argv) {// the entire difference between CUDA0's cache and CUDA1's: 814 slots against 188, 2026-09-30. Both of// those allocations are already made before a stage's cache is sized, so what has to be held back here is// the windows and - only on the stage that carries them - the drafter and the head.- const int64_t kWindowMib = 96; // the verify windows; 75 MiB measured, rounded up- const int64_t kDrafterMib = 1000; // the MTP drafter (839 MiB) + the head, on the last stage only+ // (the arithmetic is program/stage_reserve.hpp's: 96 MiB for the verify windows - 75 MiB measured, rounded up -+ // and 1000 MiB for the MTP drafter and the head on the last stage only; pinned by tests/program/stage_reserve_test.cpp)// --pipeline-windows: a second verifier on every stage (its arena alone is 70-75 MiB and its graphs come on top:// 64 MiB left two RTX 5070 Ti under Linux short of it) and, with the decode pipelined, two copies of the first// stage's GDN state (sized from its session where the first card's cache is sized; this is the split search's@@ -4653,9 +4712,14 @@ int main(int argc, char** argv) {const int64_t pf = search && split_own_auto && !place_with_reserve ? 0 : split_pf_mib;const int64_t base_reserve =later && o.vram_reserve_later_mib >= 0 ? o.vram_reserve_later_mib : o.vram_reserve_mib;- const int64_t reserve = (base_reserve + pf + (later ? kWindowMib : kPipeSnapMib) +- kPipeWindowMib + (drafter ? kDrafterMib : 0)) << 20;- return std::max<int64_t>((int64_t) fb - reserve, 0);+ strata::program::stage_reserve::Inputs rin;+ rin.base_reserve_mib = base_reserve;+ rin.prompt_buffers_mib = pf;+ rin.later = later;+ rin.drafter = drafter;+ rin.pipe_window_mib = kPipeWindowMib;+ rin.pipe_snap_mib = kPipeSnapMib;+ return strata::program::stage_reserve::room_bytes((int64_t) fb, rin);};if (multi_gpu && split_auto) {const auto& lay = strata::kernels::cpu::expert_layout();@@ -9018,6 +9082,7 @@ int main(int argc, char** argv) {if (!sp.set_peer(&peer, rows, err)) {std::fprintf(stderr, "strata serve: %s - the prompt path stays on the primary GPU\n", err.c_str());err.clear();+ (void) cudaGetLastError(); // a failed allocation here must not end the engine at the next kernel's check}}// a layer split: a one-chunk prompt runs the stages one after the other, so each stage's prompt path gets the@@ -9030,6 +9095,7 @@ int main(int argc, char** argv) {if (!paths[i].first->set_stage_helper(paths[(i + 1) % paths.size()].first, err)) {std::fprintf(stderr, "strata serve: %s - stage %zu reads its prompts alone\n", err.c_str(), i);err.clear();+ (void) cudaGetLastError(); // (as above)}}}@@ -11223,6 +11289,16 @@ int main(int argc, char** argv) {"a full card can stall a request",#endif(long long) (later_reserve + 768 - sf_mib));+ if (sf_mib < 512) { // #3702: how far the plan was off, so the reserve to ask for is not a guess+ strata::program::stage_reserve::Inputs rin;+ rin.base_reserve_mib = later_reserve;+ rin.prompt_buffers_mib = split_pf_mib;+ rin.later = true;+ rin.pipe_window_mib = kPipeWindowMib;+ const std::string why = strata::program::gpu_report::planned_vs_free_text(+ strata::program::stage_reserve::reserve_mib(rin), sf_mib);+ if (!why.empty()) std::fprintf(stderr, "strata serve: CUDA%d: %s\n", stp->dev, why.c_str());+ }}}// what the server's Monitor tab shows (servers before 0.1.8 skip unknown lines until READY)diff --git a/tests/program/gpu_report_test.cpp b/tests/program/gpu_report_test.cppnew file mode 100644index 000000000..8a64b6721--- /dev/null+++ b/tests/program/gpu_report_test.cpp@@ -0,0 +1,96 @@+// tests/program/gpu_report_test.cpp - #3797: the out-of-memory report at start (pure text; the NVML query is only+// checked to be safe to call: it must return, not throw, and leave the list empty when it answers false).+#include "strata/program/gpu_report.hpp"++#include <cstdio>+#include <string>++using namespace strata::program::gpu_report;++namespace {+int g_fail = 0;+void check(bool ok, const char* what, const std::string& got = {}) {+ if (ok) return;+ std::fprintf(stderr, "FAIL: %s%s%s\n", what, got.empty() ? "" : "\n got: ", got.c_str());+ ++g_fail;+}+bool has(const std::string& s, const char* part) { return s.find(part) != std::string::npos; }+} // namespace++int main() {+ { // the case of #3797: 320 MiB free of a 22 GB card, another program holds the rest+ Card c;+ c.index = 1;+ c.name = "NVIDIA GeForce RTX 2080 Ti";+ c.free_mib = 320;+ c.total_mib = 21998;+ c.processes_known = true;+ c.processes = {{4242, "/usr/bin/python3.12", 20100}, {777, "/opt/strata/strata", 900}, {1, "Xorg", 12}, {5, "", 0}};+ const std::string r = format_gpu_memory_report({c}, 777);+ check(has(r, "CUDA1 (NVIDIA GeForce RTX 2080 Ti): 320 MiB free of 21998 MiB"), "free and total", r);+ check(has(r, "python3.12 (pid 4242) 19.6 GiB"), "the largest other program, by file name", r);+ check(!has(r, "strata (pid 777)"), "this process is not listed as another program", r);+ check(has(r, "Xorg (pid 1) 12 MiB"), "a small one is still named", r);+ check(has(r, "1 program whose memory the driver does not report"), "a process of unknown size is counted", r);+ check(r.find("python3.12") < r.find("Xorg"), "largest first", r);+ }+ { // more than three: the rest is counted+ Card c;+ c.index = 0;+ c.free_mib = 100;+ c.total_mib = 12288;+ c.processes_known = true;+ for (uint32_t i = 1; i <= 6; ++i) c.processes.push_back({i, "p" + std::to_string(i), 100u * i});+ const std::string r = format_gpu_memory_report({c}, 0);+ check(has(r, "p6 (pid 6) 600 MiB, p5 (pid 5) 500 MiB, p4 (pid 4) 400 MiB, and 3 more"), "three named, three counted", r);+ check(!has(r, "p3 (pid 3)"), "the fourth is not named", r);+ }+ { // the driver would not say: no claim about other programs+ Card c;+ c.index = 0;+ c.free_mib = 512;+ c.total_mib = 8192;+ c.processes_known = false;+ const std::string r = format_gpu_memory_report({c}, 1);+ check(has(r, "512 MiB free of 8192 MiB; the driver does not say which programs use it here"), "unknown processes", r);+ check(!has(r, "other programs on it"), "no list", r);+ }+ { // the driver answered with nothing but ourselves: said carefully (Windows often lists no program at all)+ Card c;+ c.index = 0;+ c.free_mib = 700;+ c.total_mib = 12288;+ c.processes_known = true;+ c.processes = {{99, "strata.exe", 11000}};+ const std::string r = format_gpu_memory_report({c}, 99);+ check(has(r, "the driver lists no other program using it"), "only ourselves", r);+ }+ { // two cards, a pid with no name, a Windows path+ Card a, b;+ a.index = 0; a.free_mib = 2000; a.total_mib = 16000; a.processes_known = true;+ b.index = 1; b.free_mib = 50; b.total_mib = 12000; b.processes_known = true;+ b.processes = {{31, "", 11000}, {32, "C:\\Program Files\\Obs\\obs64.exe", 300}};+ const std::string r = format_gpu_memory_report({a, b}, 5);+ check(has(r, "CUDA0: 2000 MiB free of 16000 MiB; the driver lists no other program using it\n"), "first card line", r);+ check(has(r, "CUDA1: 50 MiB free of 12000 MiB; other programs on it: pid 31 10.7 GiB, obs64.exe (pid 32) 300 MiB"), "second card line", r);+ }+ check(format_gpu_memory_report({}, 1).empty(), "no cards, no text");+ { // planned versus free+ check(planned_vs_free_text(796, 68) ==+ "the layer split planned to keep 796 MiB free here after loading; 68 MiB are free, so 728 MiB went to something the plan did not count",+ "planned versus free", planned_vs_free_text(796, 68));+ check(planned_vs_free_text(796, 796).empty() && planned_vs_free_text(796, 900).empty(), "enough free: nothing to say");+ check(planned_vs_free_text(0, 10).empty() && planned_vs_free_text(-5, 10).empty(), "no plan, no text");+ check(has(planned_vs_free_text(300, -4), "0 MiB are free, so 300 MiB"), "a negative free count is read as 0");+ }+ { // the query never throws and never leaves a half-filled list behind a false+ std::vector<Process> p{{1, "x", 1}};+ const bool ok = query_other_processes("", p);+ check(!ok && p.empty(), "an empty bus id is refused");+ std::vector<Process> q{{1, "x", 1}};+ const bool r = query_other_processes("ffff:ff:ff.f", q); // no such card: false, or true with a list; both are fine+ check(r || q.empty(), "a failed query leaves the list empty");+ }+ std::printf(g_fail ? "FAIL\n" : "PASS\n");+ return g_fail ? 1 : 0;+}diff --git a/tests/program/stage_reserve_test.cpp b/tests/program/stage_reserve_test.cppnew file mode 100644index 000000000..355fc358f--- /dev/null+++ b/tests/program/stage_reserve_test.cpp@@ -0,0 +1,89 @@+// tests/program/stage_reserve_test.cpp - #3702: the later-stage reserve arithmetic of the layer split search.+// reserve_mib / room_bytes must equal generate.cpp's `stage_room` expression as it was written there; the proposal+// functions are pinned too (they are not used by the default placement).+#include "strata/program/stage_reserve.hpp"++#include <cstdint>+#include <cstdio>+#include <string>++using namespace strata::program::stage_reserve;++namespace {+int g_fail = 0;+void check(bool ok, const std::string& what) {+ if (ok) return;+ std::fprintf(stderr, "FAIL: %s\n", what.c_str());+ ++g_fail;+}++// the expression as it stood in generate.cpp before the move (kWindowMib, kDrafterMib, kPipeSnapMib, kPipeWindowMib)+int64_t legacy_room(int64_t fb, int64_t vram_reserve_mib, int64_t vram_reserve_later_mib, int64_t pf, bool later, bool drafter,+ int64_t pipeline_windows) {+ const int64_t kWin = 96, kDrafter = 1000;+ const int64_t kPipeWindow = pipeline_windows > 0 ? 160 : 0;+ const int64_t kPipeSnap = pipeline_windows >= 2 ? 96 : 0;+ const int64_t base_reserve = later && vram_reserve_later_mib >= 0 ? vram_reserve_later_mib : vram_reserve_mib;+ const int64_t reserve = (base_reserve + pf + (later ? kWin : kPipeSnap) + kPipeWindow + (drafter ? kDrafter : 0)) << 20;+ return std::max<int64_t>(fb - reserve, 0);+}+} // namespace++int main() {+ // the same numbers as the old expression over a grid of settings+ long cases = 0;+ for (int64_t fb : {int64_t(0), int64_t(500) << 20, int64_t(3000) << 20, int64_t(15) << 30})+ for (int64_t vr : {int64_t(0), int64_t(700), int64_t(1280)})+ for (int64_t vl : {int64_t(-1), int64_t(0), int64_t(1400)})+ for (int64_t pf : {int64_t(0), int64_t(160 + 4096 * 680 / 1024)})+ for (int later = 0; later < 2; ++later)+ for (int dr = 0; dr < 2; ++dr)+ for (int64_t pw : {int64_t(0), int64_t(1), int64_t(2)}) {+ Inputs in;+ in.base_reserve_mib = later && vl >= 0 ? vl : vr;+ in.prompt_buffers_mib = pf;+ in.later = later != 0;+ in.drafter = dr != 0;+ in.pipe_window_mib = pw > 0 ? 160 : 0;+ in.pipe_snap_mib = pw >= 2 ? 96 : 0;+ ++cases;+ check(room_bytes(fb, in) == legacy_room(fb, vr, vl, pf, later != 0, dr != 0, pw),+ "room differs from the old expression (fb " + std::to_string(fb >> 20) + " MiB, reserve " ++ std::to_string(vr) + "/" + std::to_string(vl) + ", later " + std::to_string(later) + ")");+ }+ // the configuration of #3702: default reserves (700), borrowed prompt buffers, CUDA1 = the last card with the drafter+ {+ Inputs in;+ in.base_reserve_mib = 700;+ in.later = true;+ in.drafter = true;+ check(reserve_mib(in) == 700 + 96 + 1000, "the last card of #3702 keeps 1796 MiB planned");+ in.drafter = false;+ check(reserve_mib(in) == 796, "a later card without the drafter keeps 796 MiB planned");+ in.later = false;+ check(reserve_mib(in) == 700, "the first card keeps the base reserve alone");+ in.pipe_snap_mib = 96;+ check(reserve_mib(in) == 796, "the first card books the pipeline's second GDN state");+ }+ // the same arithmetic with --vram-reserve-later-mib 1400 (the workaround that worked in #3702)+ {+ Inputs in;+ in.base_reserve_mib = 1400;+ in.later = true;+ check(reserve_mib(in) == 1496, "the workaround reserve: 1400 + 96");+ }+ // the proposal: the window term grows with the number of window sizes (--spec 4 -> windows of 1..6 tokens)+ check(proposed_window_mib(1) == 96 + 48, "proposal: one window size = the arena and the burst");+ check(proposed_window_mib(6) == 96 + 5 * 6 + 48, "proposal: six sizes = 174 MiB");+ check(proposed_window_mib(0) == proposed_window_mib(1), "proposal: no negative sizes");+ {+ Inputs in;+ in.base_reserve_mib = 700;+ in.later = true;+ check(proposed_reserve_mib(in, 6) == 700 + 174, "proposed reserve on a later card, six sizes");+ in.later = false;+ check(proposed_reserve_mib(in, 6) == reserve_mib(in), "the first card is not changed by the proposal");+ }+ std::printf("stage_reserve_test: %ld cases, %s\n", cases, g_fail ? "FAIL" : "PASS");+ return g_fail ? 1 : 0;+}