Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Later-card VRAM: audit fixes for failed allocations, the reserve arithmetic as a tested helper, a per-card out-of-memory report (#3638 #3702 #3797)

Niko12212a9a67c174e5
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 09459ac06..4a14a173b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -315,6 +315,18 @@ if(STRATA_BUILD_TESTS)
target_include_directories(split_rules_test PRIVATE include)
add_test(NAME split_rules_test COMMAND split_rules_test)
+ # #3797: the out-of-memory report at start (per card free/total, the largest other programs): the text is a pure
+ # function; the driver query (NVML, loaded at run time) is only checked to be safe to call. No GPU needed.
+ add_executable(gpu_report_test tests/program/gpu_report_test.cpp)
+ target_include_directories(gpu_report_test PRIVATE include)
+ target_link_libraries(gpu_report_test PRIVATE ${CMAKE_DL_LIBS})
+ add_test(NAME gpu_report_test COMMAND gpu_report_test)
+
+ # #3702: the later-stage reserve of the layer split search (header-only arithmetic, pinned against the old expression)
+ add_executable(stage_reserve_test tests/program/stage_reserve_test.cpp)
+ target_include_directories(stage_reserve_test PRIVATE include)
+ add_test(NAME stage_reserve_test COMMAND stage_reserve_test)
+
# the prompt planner's host-memory model (#2824): header-only, no GPU
add_executable(prefill_plan_test tests/program/prefill_plan_test.cpp)
target_include_directories(prefill_plan_test PRIVATE include)
diff --git a/include/strata/program/gpu_report.hpp b/include/strata/program/gpu_report.hpp
new file mode 100644
index 000000000..efc6f6567
--- /dev/null
+++ b/include/strata/program/gpu_report.hpp
@@ -0,0 +1,217 @@
+// include/strata/program/gpu_report.hpp - #3797: what to tell the user when a GPU allocation at start fails.
+//
+// "cudaMalloc(818947840) for the weight arena failed: 320 MiB of 21998 MiB free" says that the card is full, not by
+// whom. The usual answer is another program (or an engine that is still exiting), so the report names, per card, the
+// memory that is free and the largest other processes that hold some. The text is a warning added to the message that
+// was already there; nothing about the failure changes.
+//
+// `format_gpu_memory_report` is a pure function (tests/program/gpu_report_test.cpp). `query_other_processes` asks
+// the NVIDIA driver library (NVML, loaded at run time: no link-time dependency, so a build without it, an AMD card,
+// a missing driver library or a failed call just returns false and the report leaves the process list out). It
+// never throws and never ends the process.
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <string>
+#include <vector>
+
+#if defined(_WIN32)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+namespace strata::program::gpu_report {
+
+struct Process {
+ uint32_t pid = 0;
+ std::string name; ///< the program's file name; empty when the system would not say
+ uint64_t used_mib = 0; ///< 0 = the driver does not report it (Windows WDDM does not for most programs)
+};
+
+struct Card {
+ int index = 0; ///< the CUDA device number
+ std::string name; ///< "NVIDIA GeForce RTX 2080 Ti"; may be empty
+ uint64_t free_mib = 0;
+ uint64_t total_mib = 0;
+ bool processes_known = false; ///< the driver answered the process query (the list may still be empty)
+ std::vector<Process> processes; ///< every process that holds memory on this card, in any order
+};
+
+namespace detail {
+inline std::string size_text(uint64_t mib) {
+ char b[48];
+ if (mib >= 10240) std::snprintf(b, sizeof b, "%.1f GiB", (double) mib / 1024.0);
+ else std::snprintf(b, sizeof b, "%llu MiB", (unsigned long long) mib);
+ return b;
+}
+inline std::string base_name(const std::string& path) {
+ const size_t cut = path.find_last_of("/\\");
+ return cut == std::string::npos ? path : path.substr(cut + 1);
+}
+} // namespace detail
+
+/// One line per card: the free and total memory and, when the driver listed them, the largest other processes
+/// (`self_pid` and any process of 0 MiB that the driver does list are left out of "largest"; at most `max_procs`
+/// are named). Ends with no newline. `cards` empty: an empty string.
+inline std::string format_gpu_memory_report(const std::vector<Card>& cards, uint32_t self_pid, size_t max_procs = 3) {
+ std::string out;
+ for (const Card& c : cards) {
+ if (!out.empty()) out += "\n";
+ out += "strata generate: CUDA" + std::to_string(c.index);
+ if (!c.name.empty()) out += " (" + c.name + ")";
+ out += ": " + std::to_string((unsigned long long) c.free_mib) + " MiB free of " +
+ std::to_string((unsigned long long) c.total_mib) + " MiB"; // (as the engine's other memory lines)
+ if (!c.processes_known) {
+ out += "; the driver does not say which programs use it here";
+ continue;
+ }
+ std::vector<Process> others;
+ size_t unsized = 0;
+ for (const Process& p : c.processes) {
+ if (p.pid == self_pid) continue;
+ if (p.used_mib == 0) { ++unsized; continue; }
+ others.push_back(p);
+ }
+ std::sort(others.begin(), others.end(), [](const Process& a, const Process& b) {
+ return a.used_mib != b.used_mib ? a.used_mib > b.used_mib : a.pid < b.pid;
+ });
+ if (others.empty() && unsized == 0) {
+ out += "; the driver lists no other program using it";
+ continue;
+ }
+ out += "; other programs on it: ";
+ for (size_t i = 0; i < others.size() && i < max_procs; ++i) {
+ if (i > 0) out += ", ";
+ const std::string n = detail::base_name(others[i].name);
+ out += (n.empty() ? "pid " + std::to_string(others[i].pid) : n + " (pid " + std::to_string(others[i].pid) + ")") + " " +
+ detail::size_text(others[i].used_mib);
+ }
+ if (others.size() > max_procs) out += ", and " + std::to_string(others.size() - max_procs) + " more";
+ if (unsized > 0) out += std::string(others.empty() ? "" : "; ") + std::to_string(unsized) +
+ (unsized == 1 ? " program whose memory the driver does not report" : " programs whose memory the driver does not report");
+ }
+ return out;
+}
+
+/// "The split planned to keep N MiB free on this card, M MiB are free" for the low-VRAM warning: how far the engine's
+/// own estimate was off. Empty when the card has at least what was planned (nothing to explain).
+inline std::string planned_vs_free_text(int64_t planned_mib, int64_t free_mib) {
+ if (planned_mib <= 0 || free_mib >= planned_mib) return {};
+ return "the layer split planned to keep " + std::to_string((long long) planned_mib) + " MiB free here after loading; " +
+ std::to_string((long long) std::max<int64_t>(free_mib, 0)) + " MiB are free, so " +
+ std::to_string((long long) (planned_mib - std::max<int64_t>(free_mib, 0))) + " MiB went to something the plan did not count";
+}
+
+// ---------------------------------------------------------------------------------------------- the driver query
+namespace detail {
+struct NvmlProc { // nvmlProcessInfo_v2_t / the layout the _v3 calls fill
+ unsigned int pid;
+ unsigned long long usedGpuMemory;
+ unsigned int gpuInstanceId;
+ unsigned int computeInstanceId;
+};
+constexpr unsigned long long kNvmlNotAvailable = ~0ull;
+using NvmlInit = int (*)();
+using NvmlHandleByPci = int (*)(const char*, void**);
+using NvmlProcs = int (*)(void*, unsigned int*, NvmlProc*);
+using NvmlProcName = int (*)(unsigned int, char*, unsigned int);
+
+struct Nvml {
+ void* lib = nullptr;
+ NvmlHandleByPci by_pci = nullptr;
+ NvmlProcs compute = nullptr;
+ NvmlProcs graphics = nullptr;
+ NvmlProcName name = nullptr;
+ NvmlInit shutdown = nullptr;
+};
+
+inline void* open_lib() {
+#if defined(_WIN32)
+ return (void*) LoadLibraryA("nvml.dll");
+#else
+ for (const char* n : {"libnvidia-ml.so.1", "libnvidia-ml.so"})
+ if (void* h = dlopen(n, RTLD_LAZY | RTLD_LOCAL)) return h;
+ return nullptr;
+#endif
+}
+inline void* sym(void* lib, const char* n) {
+#if defined(_WIN32)
+ return (void*) GetProcAddress((HMODULE) lib, n);
+#else
+ return dlsym(lib, n);
+#endif
+}
+inline void close_lib(void* lib) {
+#if defined(_WIN32)
+ if (lib) FreeLibrary((HMODULE) lib);
+#else
+ if (lib) dlclose(lib);
+#endif
+}
+} // namespace detail
+
+/// The processes that hold memory on the card with this PCI bus id ("0000:65:00.0", what cudaDeviceGetPCIBusId
+/// returns). true: the driver answered (the list may be empty). false: it could not be asked. Best effort.
+inline bool query_other_processes(const char* pci_bus_id, std::vector<Process>& out) {
+ out.clear();
+ if (pci_bus_id == nullptr || pci_bus_id[0] == '\0') return false;
+ detail::Nvml n;
+ n.lib = detail::open_lib();
+ if (n.lib == nullptr) return false;
+ bool ok = false;
+ do {
+ auto init = (detail::NvmlInit) detail::sym(n.lib, "nvmlInit_v2");
+ n.shutdown = (detail::NvmlInit) detail::sym(n.lib, "nvmlShutdown");
+ n.by_pci = (detail::NvmlHandleByPci) detail::sym(n.lib, "nvmlDeviceGetHandleByPciBusId_v2");
+ n.name = (detail::NvmlProcName) detail::sym(n.lib, "nvmlSystemGetProcessName");
+ // _v3 and _v2 take the 24-byte record above; the unversioned (v1) call has another record, so it is not used
+ n.compute = (detail::NvmlProcs) detail::sym(n.lib, "nvmlDeviceGetComputeRunningProcesses_v3");
+ if (!n.compute) n.compute = (detail::NvmlProcs) detail::sym(n.lib, "nvmlDeviceGetComputeRunningProcesses_v2");
+ n.graphics = (detail::NvmlProcs) detail::sym(n.lib, "nvmlDeviceGetGraphicsRunningProcesses_v3");
+ if (!n.graphics) n.graphics = (detail::NvmlProcs) detail::sym(n.lib, "nvmlDeviceGetGraphicsRunningProcesses_v2");
+ if (!init || !n.by_pci || !n.compute || init() != 0) break;
+ void* dev = nullptr;
+ if (n.by_pci(pci_bus_id, &dev) != 0 || dev == nullptr) { if (n.shutdown) n.shutdown(); break; }
+ for (detail::NvmlProcs call : {n.compute, n.graphics}) {
+ if (!call) continue;
+ std::vector<detail::NvmlProc> recs(128);
+ unsigned int count = (unsigned int) recs.size();
+ int rc = call(dev, &count, recs.data());
+ if (rc == 7 && count > recs.size()) { // NVML_ERROR_INSUFFICIENT_SIZE: `count` is the number
+ recs.resize((size_t) count + 8);
+ count = (unsigned int) recs.size();
+ rc = call(dev, &count, recs.data());
+ }
+ if (rc != 0) continue;
+ ok = true;
+ for (unsigned int i = 0; i < count && i < recs.size(); ++i) {
+ bool seen = false;
+ for (Process& p : out)
+ if (p.pid == recs[i].pid) { seen = true; break; }
+ if (seen) continue;
+ Process p;
+ p.pid = recs[i].pid;
+ p.used_mib = recs[i].usedGpuMemory == detail::kNvmlNotAvailable ? 0 : (uint64_t) (recs[i].usedGpuMemory >> 20);
+ if (n.name) {
+ char buf[260] = {};
+ if (n.name(p.pid, buf, (unsigned int) sizeof buf) == 0) p.name = buf;
+ }
+ out.push_back(std::move(p));
+ }
+ }
+ if (n.shutdown) n.shutdown();
+ } while (false);
+ detail::close_lib(n.lib);
+ return ok;
+}
+
+} // namespace strata::program::gpu_report
diff --git a/include/strata/program/stage_reserve.hpp b/include/strata/program/stage_reserve.hpp
new file mode 100644
index 000000000..f76435303
--- /dev/null
+++ b/include/strata/program/stage_reserve.hpp
@@ -0,0 +1,61 @@
+// include/strata/program/stage_reserve.hpp - what the layer split keeps free on a card when it sizes the expert cache.
+//
+// This is the arithmetic of `stage_room` in src/program/generate.cpp, moved here unchanged so a host test can pin it
+// (tests/program/stage_reserve_test.cpp). `reserve_mib` is exactly the sum that function subtracted from the card's
+// free memory before; the default placement does not change.
+//
+// #3702: on an unequal pair (RTX 3060 + RTX 5060 Ti) the last card was left with 68 MiB after loading although
+// 700 + 96 MiB were reserved for it, and the verify windows' graphs (captured on first use, 4-6 MiB each and once 42 MiB)
+// then ran it out of memory. `proposed_window_mib` below is a PROPOSAL for a larger window term; nothing calls it
+// (it needs the model gate in phaseA-tests/gh027/gates/3702-later-reserve.sh before it can become the default).
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+
+namespace strata::program::stage_reserve {
+
+constexpr int64_t kWindowMib = 96; ///< the verify windows on a later stage; 75 MiB measured, rounded up
+constexpr int64_t kDrafterMib = 1000; ///< the MTP drafter (839 MiB) + the head, on the last stage only
+
+struct Inputs {
+ int64_t base_reserve_mib = 0; ///< --vram-reserve-later-mib on a later stage when given, else --vram-reserve-mib
+ int64_t prompt_buffers_mib = 0; ///< the prompt path's own buffers (0 when they are borrowed from the cache)
+ bool later = false; ///< a stage after the first
+ bool drafter = false; ///< the stage that carries the MTP drafter and the head
+ int64_t pipe_window_mib = 0; ///< --pipeline-windows: a second verifier on every stage
+ int64_t pipe_snap_mib = 0; ///< --pipeline-windows 2 or more: the first stage's second GDN state
+};
+
+/// What the plan keeps free on the card, in MiB (generate.cpp's `reserve` before the shift).
+inline int64_t reserve_mib(const Inputs& in) {
+ return in.base_reserve_mib + in.prompt_buffers_mib + (in.later ? kWindowMib : in.pipe_snap_mib) + in.pipe_window_mib +
+ (in.drafter ? kDrafterMib : 0);
+}
+
+/// The room the cache may use: the free bytes less the reserve, never below 0.
+inline int64_t room_bytes(int64_t free_bytes, const Inputs& in) {
+ return std::max<int64_t>(free_bytes - (reserve_mib(in) << 20), 0);
+}
+
+// ---------------------------------------------------------------------------------------------------- the proposal
+constexpr int64_t kGraphMib = 6; ///< one window graph, captured on first use (log of #3702: 4, 6, 6)
+constexpr int64_t kInstantiateBurstMib = 48; ///< the largest single step seen (42 MiB in #3702): room kept for one more
+
+/// PROPOSAL, unused: the window term of a later stage when the windows of every size 1..max_window_t can still be
+/// captured after loading: the first window's arena (kWindowMib already holds it), a graph for each further size, and
+/// one burst.
+inline int64_t proposed_window_mib(int max_window_t) {
+ const int64_t sizes = std::max<int64_t>(max_window_t, 1);
+ return kWindowMib + (sizes - 1) * kGraphMib + kInstantiateBurstMib;
+}
+
+/// PROPOSAL, unused: reserve_mib with the proposed window term on a later stage.
+inline int64_t proposed_reserve_mib(const Inputs& in, int max_window_t) {
+ Inputs p = in;
+ int64_t r = reserve_mib(p);
+ if (in.later) r += proposed_window_mib(max_window_t) - kWindowMib;
+ return r;
+}
+
+} // namespace strata::program::stage_reserve
diff --git a/src/core/mtp.cpp b/src/core/mtp.cpp
index 292a5d2b4..972006137 100644
--- a/src/core/mtp.cpp
+++ b/src/core/mtp.cpp
@@ -1238,16 +1238,19 @@ bool finish_capture(cudaStream_t cs, bool ok, cudaGraphExec_t& exec, const char*
if (graph) cudaGraphDestroy(graph);
return false;
}
- if (ce != cudaSuccess || cudaGraphInstantiate(&exec, graph, 0) != cudaSuccess) {
+ cudaGraphExec_t got = nullptr; // (a failed instantiate leaves nothing in the member)
+ const cudaError_t ie = ce != cudaSuccess ? ce : cudaGraphInstantiate(&got, graph, 0);
+ if (ie != cudaSuccess) {
if (graph) cudaGraphDestroy(graph);
- err = std::string("mtp: ") + what + " capture: " + cudaGetErrorString(ce);
+ err = std::string("mtp: ") + what + " capture: " + cudaGetErrorString(ie); // the status that failed
return false;
}
+ exec = got;
mtp_list_nodes(graph, what);
cudaGraphDestroy(graph);
- // an explicit upload: the first launch's implicit one blocked behind a device-side spin (verify.cpp)
- cudaGraphUpload(exec, cs);
- cudaStreamSynchronize(cs);
+ // an explicit upload: the first launch's implicit one blocked behind a device-side spin (verify.cpp); only a warm-up,
+ // so a failure (no VRAM left) is cleared and the first launch uploads it
+ if (cudaGraphUpload(exec, cs) != cudaSuccess || cudaStreamSynchronize(cs) != cudaSuccess) (void) cudaGetLastError();
return true;
}
} // namespace
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index 5e97b8f52..193b8f5ba 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -2583,6 +2583,12 @@ bool Verifier::capture(int T, std::string& err) {
const cudaError_t us = cudaStreamSynchronize(cs_);
std::fprintf(stderr, "strata verify: captured the %d-token window (upload %s, sync %s)\n", T,
cudaGetErrorString(ue), cudaGetErrorString(us));
+ // The upload only warms the graph; the first launch does it otherwise. A failed one (out of memory on a full card)
+ // must not stay pending: the next kernel wrapper's cudaGetLastError would read it and end the engine (#3702).
+ if (ue != cudaSuccess || us != cudaSuccess) {
+ (void) cudaGetLastError();
+ std::fprintf(stderr, "strata verify: the %d-token window's upload was skipped (the first launch does it)\n", T);
+ }
return true;
}
@@ -2706,13 +2712,16 @@ bool Verifier::capture_commit(std::string& err) {
if (graph) cudaGraphDestroy(graph);
return false;
}
- if (ce != cudaSuccess || cudaGraphInstantiate(&commit_exec_, graph, 0) != cudaSuccess) {
+ if (ce != cudaSuccess) {
if (graph) cudaGraphDestroy(graph);
err = std::string("verify: commit capture: ") + cudaGetErrorString(ce);
return false;
}
+ // (through the same path as the window graphs: on a full card it frees older batch layouts and retries, and the error
+ // it reports is the instantiate's own)
+ const bool made = instantiate_evicting(commit_exec_, graph, {}, "verify: commit instantiate: ", err, 0);
cudaGraphDestroy(graph);
- return true;
+ return made;
}
void Verifier::stage_inputs(int T, const int32_t* tokens, int64_t pos0) {
@@ -3545,8 +3554,8 @@ bool Verifier::capture_batch(const int* rows, int S, int hbase, std::string& err
const bool made = instantiate_evicting(ex, graph, bkey(rows, S, hbase), "verify: batch instantiate: ", err);
cudaGraphDestroy(graph);
if (!made) return false;
- cudaGraphUpload(ex, cs_);
- cudaStreamSynchronize(cs_);
+ if (cudaGraphUpload(ex, cs_) != cudaSuccess || cudaStreamSynchronize(cs_) != cudaSuccess)
+ (void) cudaGetLastError(); // only a warm-up: the first launch uploads it (see capture)
std::string list;
for (int t = 0; t < S; ++t) list += (t ? "," : "") + std::to_string(rows[t]);
std::fprintf(stderr, "strata verify: captured the batch window over slots %s\n", list.c_str());
@@ -3720,14 +3729,17 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, cons
// #997: every layout of active slots keeps its own graph pair (20-30 MiB each on an L40S), and with an expert cache
// sized to the reserve the next window graph - a batch layout or a one-request window captured on first use - can
// find no VRAM left: free the least recently used layouts until it fits (they are captured again when needed)
- cudaError_t ie = cudaGraphInstantiate(&ex, graph, 0);
+ // (into a local: what a failed instantiate leaves in the output is not specified, and `ex` is a member that
+ // capture() reads as "captured" when it is not null)
+ cudaGraphExec_t got = nullptr;
+ cudaError_t ie = cudaGraphInstantiate(&got, graph, 0);
int freed = 0;
for (bool evicted = true; ie == cudaErrorMemoryAllocation && evicted;) {
(void) cudaGetLastError();
if (!evict_batch_graph(key, evicted, err)) return false;
if (evicted) {
++freed;
- ie = cudaGraphInstantiate(&ex, graph, 0);
+ ie = cudaGraphInstantiate(&got, graph, 0);
}
}
if (freed > 0)
@@ -3740,7 +3752,7 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, cons
if (!evict_window_graph(keep_t, evicted, err)) return false;
if (evicted) {
++freed_sizes;
- ie = cudaGraphInstantiate(&ex, graph, 0);
+ ie = cudaGraphInstantiate(&got, graph, 0);
}
}
if (freed_sizes > 0)
@@ -3748,9 +3760,11 @@ bool Verifier::instantiate_evicting(cudaGraphExec_t& ex, cudaGraph_t graph, cons
"sizes (they are captured again when used; a larger --vram-reserve-mib or "
"--vram-reserve-later-mib keeps them)\n", keep_t, freed_sizes);
if (ie != cudaSuccess) {
+ got = nullptr;
err = std::string(what) + cudaGetErrorString(ie);
return false;
}
+ ex = got;
return true;
}
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 08b529e6f..e253806f7 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -1600,6 +1600,9 @@ bool Prefill::carve(size_t T, void* alloc) {
const uint64_t region = std::max({gdn_set_bytes(T), qsa_set_bytes(T, m.cap, m.max_blocks, m.sel_batch,
m.attn_batch, s), moe_set_bytes(T, m.g->n_expert, fz)});
uint8_t* base = o.take<uint8_t>((size_t) region, ok);
+ // No region: the sub-allocators below would each cudaMalloc their share (hundreds of MiB) on a card that has just
+ // shown it has no room, and leave those buffers in `owned`. The failure is reported by the caller.
+ if (base == nullptr) return false;
m.region = base;
m.region_bytes = region;
Alloc a;
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 75d404ac9..1adb7e183 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -84,6 +84,8 @@
#include "strata/program/logits_selection.hpp"
#include "strata/program/conv_cache.hpp"
#include "strata/program/split_rules.hpp"
+#include "strata/program/stage_reserve.hpp"
+#include "strata/program/gpu_report.hpp"
#include "strata/prefill/share_rules.hpp"
#include "strata/platform/host_limits.hpp"
#include "strata/program/split_ring.hpp"
@@ -1593,6 +1595,49 @@ int64_t res_check(const int32_t* d_res, const std::vector<int32_t>& host_res, st
return bad;
}
+/// #3797: when the engine cannot start for lack of GPU memory, the cards' free and total memory and the largest other
+/// programs that hold some, as a warning added to the message that says it (program/gpu_report.hpp). Best effort: a
+/// card that cannot be asked, or a driver that does not list its programs, is left out or said so; it never changes the
+/// failure and never throws. `devices`: the CUDA ordinals to report (duplicates are skipped).
+static void print_startup_gpu_report(const std::vector<int>& devices) {
+ namespace gr = strata::program::gpu_report;
+ std::vector<gr::Card> cards;
+ int count = 0, prev = 0;
+ if (cudaGetDeviceCount(&count) != cudaSuccess) { (void) cudaGetLastError(); return; }
+ if (cudaGetDevice(&prev) != cudaSuccess) { (void) cudaGetLastError(); prev = 0; }
+ std::vector<int> seen;
+ for (const int d : devices) {
+ if (d < 0 || d >= count || std::find(seen.begin(), seen.end(), d) != seen.end()) continue;
+ seen.push_back(d);
+ if (cudaSetDevice(d) != cudaSuccess) { (void) cudaGetLastError(); continue; } // no context fits there either
+ size_t free_b = 0, total_b = 0;
+ if (cudaMemGetInfo(&free_b, &total_b) != cudaSuccess) { (void) cudaGetLastError(); continue; }
+ gr::Card c;
+ c.index = d;
+ c.free_mib = (uint64_t) (free_b >> 20);
+ c.total_mib = (uint64_t) (total_b >> 20);
+ cudaDeviceProp prop{};
+ if (cudaGetDeviceProperties(&prop, d) == cudaSuccess) c.name = prop.name;
+ else (void) cudaGetLastError();
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906) && !defined(STRATA_GPU_METAL)
+ char bus[64] = {};
+ if (cudaDeviceGetPCIBusId(bus, (int) sizeof bus, d) == cudaSuccess) c.processes_known = gr::query_other_processes(bus, c.processes);
+ else (void) cudaGetLastError();
+#endif
+ cards.push_back(std::move(c));
+ }
+ if (prev >= 0 && prev < count) (void) cudaSetDevice(prev);
+ (void) cudaGetLastError();
+ if (cards.empty()) return;
+#if defined(_WIN32)
+ const uint32_t self = (uint32_t) GetCurrentProcessId();
+#else
+ const uint32_t self = (uint32_t) getpid();
+#endif
+ std::fprintf(stderr, "strata generate: GPU memory right now (best effort):\n%s\n",
+ gr::format_gpu_memory_report(cards, self).c_str());
+}
+
/// The pre-launch hook's patch: the changed entries of the host's residency table (`idx`) as (index, entry) pairs into
/// the mapped `pairs` (room for `cap`); their count, or -1 when they do not fit (then the whole table goes up).
int64_t res_patch_pairs(const std::vector<int32_t>& idx, const std::vector<int32_t>& host_res, int32_t* pairs, size_t cap) {
@@ -3920,6 +3965,13 @@ int main(int argc, char** argv) {
"nvidia-smi / rocm-smi lists them\n",
(unsigned long long) pool_bytes, cudaGetErrorString(ce), (unsigned long long) (free_b >> 20),
(unsigned long long) (total_b >> 20));
+ { // #3797: every card of the run, and who else is on it
+ std::vector<int> report_devs;
+ int cur = 0;
+ if (cudaGetDevice(&cur) == cudaSuccess) report_devs.push_back(cur); else (void) cudaGetLastError();
+ for (const int d : split_devs) report_devs.push_back(d);
+ print_startup_gpu_report(report_devs);
+ }
return 1;
}
mem_mark("the CUDA context and the weight arena");
@@ -4417,6 +4469,13 @@ int main(int argc, char** argv) {
err.empty() ? "the weight arena does not fit" : err.c_str(),
(unsigned long long) (pool_s >> 20), (unsigned long long) (free_b >> 20),
(unsigned long long) (total_b >> 20));
+ { // #3797: every card of the run, and who else is on it
+ std::vector<int> report_devs;
+ int cur = 0;
+ if (cudaGetDevice(&cur) == cudaSuccess) report_devs.push_back(cur); else (void) cudaGetLastError();
+ for (const int d : split_devs) report_devs.push_back(d);
+ print_startup_gpu_report(report_devs);
+ }
return false;
}
if (!o.native_dense_gguf.empty() && !st.dense.load(o.native_dense_gguf, st.wt, err, o.native_ple_key)) {
@@ -4592,8 +4651,8 @@ int main(int argc, char** argv) {
// the entire difference between CUDA0's cache and CUDA1's: 814 slots against 188, 2026-09-30. Both of
// those allocations are already made before a stage's cache is sized, so what has to be held back here is
// the windows and - only on the stage that carries them - the drafter and the head.
- const int64_t kWindowMib = 96; // the verify windows; 75 MiB measured, rounded up
- const int64_t kDrafterMib = 1000; // the MTP drafter (839 MiB) + the head, on the last stage only
+ // (the arithmetic is program/stage_reserve.hpp's: 96 MiB for the verify windows - 75 MiB measured, rounded up -
+ // and 1000 MiB for the MTP drafter and the head on the last stage only; pinned by tests/program/stage_reserve_test.cpp)
// --pipeline-windows: a second verifier on every stage (its arena alone is 70-75 MiB and its graphs come on top:
// 64 MiB left two RTX 5070 Ti under Linux short of it) and, with the decode pipelined, two copies of the first
// stage's GDN state (sized from its session where the first card's cache is sized; this is the split search's
@@ -4653,9 +4712,14 @@ int main(int argc, char** argv) {
const int64_t pf = search && split_own_auto && !place_with_reserve ? 0 : split_pf_mib;
const int64_t base_reserve =
later && o.vram_reserve_later_mib >= 0 ? o.vram_reserve_later_mib : o.vram_reserve_mib;
- const int64_t reserve = (base_reserve + pf + (later ? kWindowMib : kPipeSnapMib) +
- kPipeWindowMib + (drafter ? kDrafterMib : 0)) << 20;
- return std::max<int64_t>((int64_t) fb - reserve, 0);
+ strata::program::stage_reserve::Inputs rin;
+ rin.base_reserve_mib = base_reserve;
+ rin.prompt_buffers_mib = pf;
+ rin.later = later;
+ rin.drafter = drafter;
+ rin.pipe_window_mib = kPipeWindowMib;
+ rin.pipe_snap_mib = kPipeSnapMib;
+ return strata::program::stage_reserve::room_bytes((int64_t) fb, rin);
};
if (multi_gpu && split_auto) {
const auto& lay = strata::kernels::cpu::expert_layout();
@@ -9018,6 +9082,7 @@ int main(int argc, char** argv) {
if (!sp.set_peer(&peer, rows, err)) {
std::fprintf(stderr, "strata serve: %s - the prompt path stays on the primary GPU\n", err.c_str());
err.clear();
+ (void) cudaGetLastError(); // a failed allocation here must not end the engine at the next kernel's check
}
}
// a layer split: a one-chunk prompt runs the stages one after the other, so each stage's prompt path gets the
@@ -9030,6 +9095,7 @@ int main(int argc, char** argv) {
if (!paths[i].first->set_stage_helper(paths[(i + 1) % paths.size()].first, err)) {
std::fprintf(stderr, "strata serve: %s - stage %zu reads its prompts alone\n", err.c_str(), i);
err.clear();
+ (void) cudaGetLastError(); // (as above)
}
}
}
@@ -11223,6 +11289,16 @@ int main(int argc, char** argv) {
"a full card can stall a request",
#endif
(long long) (later_reserve + 768 - sf_mib));
+ if (sf_mib < 512) { // #3702: how far the plan was off, so the reserve to ask for is not a guess
+ strata::program::stage_reserve::Inputs rin;
+ rin.base_reserve_mib = later_reserve;
+ rin.prompt_buffers_mib = split_pf_mib;
+ rin.later = true;
+ rin.pipe_window_mib = kPipeWindowMib;
+ const std::string why = strata::program::gpu_report::planned_vs_free_text(
+ strata::program::stage_reserve::reserve_mib(rin), sf_mib);
+ if (!why.empty()) std::fprintf(stderr, "strata serve: CUDA%d: %s\n", stp->dev, why.c_str());
+ }
}
}
// what the server's Monitor tab shows (servers before 0.1.8 skip unknown lines until READY)
diff --git a/tests/program/gpu_report_test.cpp b/tests/program/gpu_report_test.cpp
new file mode 100644
index 000000000..8a64b6721
--- /dev/null
+++ b/tests/program/gpu_report_test.cpp
@@ -0,0 +1,96 @@
+// tests/program/gpu_report_test.cpp - #3797: the out-of-memory report at start (pure text; the NVML query is only
+// checked to be safe to call: it must return, not throw, and leave the list empty when it answers false).
+#include "strata/program/gpu_report.hpp"
+
+#include <cstdio>
+#include <string>
+
+using namespace strata::program::gpu_report;
+
+namespace {
+int g_fail = 0;
+void check(bool ok, const char* what, const std::string& got = {}) {
+ if (ok) return;
+ std::fprintf(stderr, "FAIL: %s%s%s\n", what, got.empty() ? "" : "\n got: ", got.c_str());
+ ++g_fail;
+}
+bool has(const std::string& s, const char* part) { return s.find(part) != std::string::npos; }
+} // namespace
+
+int main() {
+ { // the case of #3797: 320 MiB free of a 22 GB card, another program holds the rest
+ Card c;
+ c.index = 1;
+ c.name = "NVIDIA GeForce RTX 2080 Ti";
+ c.free_mib = 320;
+ c.total_mib = 21998;
+ c.processes_known = true;
+ c.processes = {{4242, "/usr/bin/python3.12", 20100}, {777, "/opt/strata/strata", 900}, {1, "Xorg", 12}, {5, "", 0}};
+ const std::string r = format_gpu_memory_report({c}, 777);
+ check(has(r, "CUDA1 (NVIDIA GeForce RTX 2080 Ti): 320 MiB free of 21998 MiB"), "free and total", r);
+ check(has(r, "python3.12 (pid 4242) 19.6 GiB"), "the largest other program, by file name", r);
+ check(!has(r, "strata (pid 777)"), "this process is not listed as another program", r);
+ check(has(r, "Xorg (pid 1) 12 MiB"), "a small one is still named", r);
+ check(has(r, "1 program whose memory the driver does not report"), "a process of unknown size is counted", r);
+ check(r.find("python3.12") < r.find("Xorg"), "largest first", r);
+ }
+ { // more than three: the rest is counted
+ Card c;
+ c.index = 0;
+ c.free_mib = 100;
+ c.total_mib = 12288;
+ c.processes_known = true;
+ for (uint32_t i = 1; i <= 6; ++i) c.processes.push_back({i, "p" + std::to_string(i), 100u * i});
+ const std::string r = format_gpu_memory_report({c}, 0);
+ check(has(r, "p6 (pid 6) 600 MiB, p5 (pid 5) 500 MiB, p4 (pid 4) 400 MiB, and 3 more"), "three named, three counted", r);
+ check(!has(r, "p3 (pid 3)"), "the fourth is not named", r);
+ }
+ { // the driver would not say: no claim about other programs
+ Card c;
+ c.index = 0;
+ c.free_mib = 512;
+ c.total_mib = 8192;
+ c.processes_known = false;
+ const std::string r = format_gpu_memory_report({c}, 1);
+ check(has(r, "512 MiB free of 8192 MiB; the driver does not say which programs use it here"), "unknown processes", r);
+ check(!has(r, "other programs on it"), "no list", r);
+ }
+ { // the driver answered with nothing but ourselves: said carefully (Windows often lists no program at all)
+ Card c;
+ c.index = 0;
+ c.free_mib = 700;
+ c.total_mib = 12288;
+ c.processes_known = true;
+ c.processes = {{99, "strata.exe", 11000}};
+ const std::string r = format_gpu_memory_report({c}, 99);
+ check(has(r, "the driver lists no other program using it"), "only ourselves", r);
+ }
+ { // two cards, a pid with no name, a Windows path
+ Card a, b;
+ a.index = 0; a.free_mib = 2000; a.total_mib = 16000; a.processes_known = true;
+ b.index = 1; b.free_mib = 50; b.total_mib = 12000; b.processes_known = true;
+ b.processes = {{31, "", 11000}, {32, "C:\\Program Files\\Obs\\obs64.exe", 300}};
+ const std::string r = format_gpu_memory_report({a, b}, 5);
+ check(has(r, "CUDA0: 2000 MiB free of 16000 MiB; the driver lists no other program using it\n"), "first card line", r);
+ check(has(r, "CUDA1: 50 MiB free of 12000 MiB; other programs on it: pid 31 10.7 GiB, obs64.exe (pid 32) 300 MiB"), "second card line", r);
+ }
+ check(format_gpu_memory_report({}, 1).empty(), "no cards, no text");
+ { // planned versus free
+ check(planned_vs_free_text(796, 68) ==
+ "the layer split planned to keep 796 MiB free here after loading; 68 MiB are free, so 728 MiB went to something the plan did not count",
+ "planned versus free", planned_vs_free_text(796, 68));
+ check(planned_vs_free_text(796, 796).empty() && planned_vs_free_text(796, 900).empty(), "enough free: nothing to say");
+ check(planned_vs_free_text(0, 10).empty() && planned_vs_free_text(-5, 10).empty(), "no plan, no text");
+ check(has(planned_vs_free_text(300, -4), "0 MiB are free, so 300 MiB"), "a negative free count is read as 0");
+ }
+ { // the query never throws and never leaves a half-filled list behind a false
+ std::vector<Process> p{{1, "x", 1}};
+ const bool ok = query_other_processes("", p);
+ check(!ok && p.empty(), "an empty bus id is refused");
+ std::vector<Process> q{{1, "x", 1}};
+ const bool r = query_other_processes("ffff:ff:ff.f", q); // no such card: false, or true with a list; both are fine
+ check(r || q.empty(), "a failed query leaves the list empty");
+ }
+ std::printf(g_fail ? "FAIL\n" : "PASS\n");
+ return g_fail ? 1 : 0;
+}
diff --git a/tests/program/stage_reserve_test.cpp b/tests/program/stage_reserve_test.cpp
new file mode 100644
index 000000000..355fc358f
--- /dev/null
+++ b/tests/program/stage_reserve_test.cpp
@@ -0,0 +1,89 @@
+// tests/program/stage_reserve_test.cpp - #3702: the later-stage reserve arithmetic of the layer split search.
+// reserve_mib / room_bytes must equal generate.cpp's `stage_room` expression as it was written there; the proposal
+// functions are pinned too (they are not used by the default placement).
+#include "strata/program/stage_reserve.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <string>
+
+using namespace strata::program::stage_reserve;
+
+namespace {
+int g_fail = 0;
+void check(bool ok, const std::string& what) {
+ if (ok) return;
+ std::fprintf(stderr, "FAIL: %s\n", what.c_str());
+ ++g_fail;
+}
+
+// the expression as it stood in generate.cpp before the move (kWindowMib, kDrafterMib, kPipeSnapMib, kPipeWindowMib)
+int64_t legacy_room(int64_t fb, int64_t vram_reserve_mib, int64_t vram_reserve_later_mib, int64_t pf, bool later, bool drafter,
+ int64_t pipeline_windows) {
+ const int64_t kWin = 96, kDrafter = 1000;
+ const int64_t kPipeWindow = pipeline_windows > 0 ? 160 : 0;
+ const int64_t kPipeSnap = pipeline_windows >= 2 ? 96 : 0;
+ const int64_t base_reserve = later && vram_reserve_later_mib >= 0 ? vram_reserve_later_mib : vram_reserve_mib;
+ const int64_t reserve = (base_reserve + pf + (later ? kWin : kPipeSnap) + kPipeWindow + (drafter ? kDrafter : 0)) << 20;
+ return std::max<int64_t>(fb - reserve, 0);
+}
+} // namespace
+
+int main() {
+ // the same numbers as the old expression over a grid of settings
+ long cases = 0;
+ for (int64_t fb : {int64_t(0), int64_t(500) << 20, int64_t(3000) << 20, int64_t(15) << 30})
+ for (int64_t vr : {int64_t(0), int64_t(700), int64_t(1280)})
+ for (int64_t vl : {int64_t(-1), int64_t(0), int64_t(1400)})
+ for (int64_t pf : {int64_t(0), int64_t(160 + 4096 * 680 / 1024)})
+ for (int later = 0; later < 2; ++later)
+ for (int dr = 0; dr < 2; ++dr)
+ for (int64_t pw : {int64_t(0), int64_t(1), int64_t(2)}) {
+ Inputs in;
+ in.base_reserve_mib = later && vl >= 0 ? vl : vr;
+ in.prompt_buffers_mib = pf;
+ in.later = later != 0;
+ in.drafter = dr != 0;
+ in.pipe_window_mib = pw > 0 ? 160 : 0;
+ in.pipe_snap_mib = pw >= 2 ? 96 : 0;
+ ++cases;
+ check(room_bytes(fb, in) == legacy_room(fb, vr, vl, pf, later != 0, dr != 0, pw),
+ "room differs from the old expression (fb " + std::to_string(fb >> 20) + " MiB, reserve " +
+ std::to_string(vr) + "/" + std::to_string(vl) + ", later " + std::to_string(later) + ")");
+ }
+ // the configuration of #3702: default reserves (700), borrowed prompt buffers, CUDA1 = the last card with the drafter
+ {
+ Inputs in;
+ in.base_reserve_mib = 700;
+ in.later = true;
+ in.drafter = true;
+ check(reserve_mib(in) == 700 + 96 + 1000, "the last card of #3702 keeps 1796 MiB planned");
+ in.drafter = false;
+ check(reserve_mib(in) == 796, "a later card without the drafter keeps 796 MiB planned");
+ in.later = false;
+ check(reserve_mib(in) == 700, "the first card keeps the base reserve alone");
+ in.pipe_snap_mib = 96;
+ check(reserve_mib(in) == 796, "the first card books the pipeline's second GDN state");
+ }
+ // the same arithmetic with --vram-reserve-later-mib 1400 (the workaround that worked in #3702)
+ {
+ Inputs in;
+ in.base_reserve_mib = 1400;
+ in.later = true;
+ check(reserve_mib(in) == 1496, "the workaround reserve: 1400 + 96");
+ }
+ // the proposal: the window term grows with the number of window sizes (--spec 4 -> windows of 1..6 tokens)
+ check(proposed_window_mib(1) == 96 + 48, "proposal: one window size = the arena and the burst");
+ check(proposed_window_mib(6) == 96 + 5 * 6 + 48, "proposal: six sizes = 174 MiB");
+ check(proposed_window_mib(0) == proposed_window_mib(1), "proposal: no negative sizes");
+ {
+ Inputs in;
+ in.base_reserve_mib = 700;
+ in.later = true;
+ check(proposed_reserve_mib(in, 6) == 700 + 174, "proposed reserve on a later card, six sizes");
+ in.later = false;
+ check(proposed_reserve_mib(in, 6) == reserve_mib(in), "the first card is not changed by the proposal");
+ }
+ std::printf("stage_reserve_test: %ld cases, %s\n", cases, g_fail ? "FAIL" : "PASS");
+ return g_fail ? 1 : 0;
+}