Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Qwen: the automatic expert cache leaves the reserve it was asked to keep - the ~245 MiB the load allocates after it are priced (#4474)

Niko1221cfa197e21a32
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ee0599211..6d1c9a858 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1633,6 +1633,11 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_include_directories(ple_residency_test PRIVATE include)
add_test(NAME ple_residency_test COMMAND ple_residency_test)
+ # #4474: the automatic expert cache's slot rule (the late allocation priced); header-only, needs nothing
+ add_executable(auto_cache_test src/core/auto_cache_test.cpp)
+ target_include_directories(auto_cache_test PRIVATE include)
+ add_test(NAME auto_cache_test COMMAND auto_cache_test)
+
# 0.2.7 (#3911): the qwen35 host sampler's penalties on a short (draft-vocab) logit row; header-only, CPU
add_executable(qwen35_sampling_test src/qwen35/sampling_test.cpp)
target_include_directories(qwen35_sampling_test PRIVATE include)
diff --git a/include/strata/core/auto_cache.hpp b/include/strata/core/auto_cache.hpp
new file mode 100644
index 000000000..9b7249e81
--- /dev/null
+++ b/include/strata/core/auto_cache.hpp
@@ -0,0 +1,47 @@
+// include/strata/core/auto_cache.hpp - the slot count of the automatic expert cache (--expert-cache auto), as a rule with a test.
+//
+// The auto cache is sized from the VRAM free after the weights, the session and the drafter, minus --vram-reserve-mib (the reserve is what
+// the operator wants LEFT once everything is loaded), the draft head that binds later (priced), the prompt path's own buffers when it does
+// not borrow cache slots (priced) and the second verifier of --pipeline-windows (priced). What was never priced is what the load allocates
+// AFTER the cache whatever the settings: the CUDA libraries the prompt path and the verifier warm up (cuBLAS handles and workspaces, the
+// lazily loaded modules) and the verifier's small buffers. Measured with STRATA_TRACE=1 on the same build, config and flags
+// (--vram-reserve-mib 700, 256K context, MTP, 4 drafts):
+// RTX A4000 16 GB (z440, IQ3_XXS): 883 MiB free after the cache -> 719 after the head and the prompt path (-164) -> 461 after the
+// verifier and the drafter's binding (-258 = 178 MiB draft head, priced, + 80) = 244 MiB unpriced
+// RTX A6000 48 GB (boston, IQ3_S): 919 -> 755 (-164) -> 459 (-296 = 213 MiB draft head, priced, + 83) = 247 MiB unpriced
+// --prefill 512 / 2048 / auto on the A4000: the same 164 + 80. v0.2.6 ends at 461 MiB on the same card: not new, but every 16 GB card with
+// the default 700 MiB ends under the 512 MiB line the engine itself warns about (#2065, #4474), and the warning cannot be cured by the
+// operator raising the reserve by the advice's figure without losing the cache it was meant to keep.
+// So the reserve is now honest: the cache leaves `reserve + kLateAllocMib`, and the card ends with about the reserve.
+#pragma once
+
+#include <cstdint>
+
+namespace strata::core {
+
+/// MiB the load allocates after the cache that no setting prices (see above; measured 244-247, a little margin).
+constexpr int64_t kLateAllocMib = 256;
+
+struct AutoCacheSlots {
+ int64_t slots = 0; ///< the cache's slot count (of the largest blob), >= 0
+ bool late_priced = false; ///< kLateAllocMib was left out of the cache
+};
+
+/// `free_b`: VRAM free now; `reserve_b`: the reserve and everything else that is priced (bytes); `blob`: the largest slot; `min_slots`: the least a working
+/// cache needs (#496). `late_mib` 0 turns the late allocation off (STRATA_DEFAULTS=026). The late allocation is only left out when the cache still has
+/// `min_slots` after it: a small card keeps the sizing it had (its reserve is lowered further elsewhere, #496), never a smaller cache than it can work with.
+inline AutoCacheSlots auto_cache_slots(int64_t free_b, int64_t reserve_b, int64_t blob, int64_t min_slots, int64_t late_mib = kLateAllocMib) {
+ AutoCacheSlots r;
+ if (blob <= 0) return r;
+ const int64_t plain = (free_b - reserve_b) / blob;
+ const int64_t late = late_mib > 0 ? (free_b - reserve_b - (late_mib << 20)) / blob : plain;
+ if (late_mib > 0 && late >= min_slots && late > 0) {
+ r.slots = late;
+ r.late_priced = true;
+ } else {
+ r.slots = plain > 0 ? plain : 0;
+ }
+ return r;
+}
+
+} // namespace strata::core
diff --git a/src/core/auto_cache_test.cpp b/src/core/auto_cache_test.cpp
new file mode 100644
index 000000000..c9ee295f2
--- /dev/null
+++ b/src/core/auto_cache_test.cpp
@@ -0,0 +1,43 @@
+#include "strata/core/auto_cache.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+
+using namespace strata::core;
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+ ++checks;
+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+constexpr int64_t MiB = 1ll << 20;
+}
+
+int main() {
+ const int64_t blob = 2560ll * 1024; // 2.5 MiB
+ const int64_t min_slots = 300;
+ // the A4000 of #4474: 9951 MiB free after the weights, 700 reserved + 184 draft head
+ const int64_t free_b = 9951 * MiB, reserve_b = (700 + 184) * MiB;
+ const AutoCacheSlots a = auto_cache_slots(free_b, reserve_b, blob, min_slots);
+ const AutoCacheSlots plain = auto_cache_slots(free_b, reserve_b, blob, min_slots, 0);
+ check(a.late_priced && !plain.late_priced, "the late allocation is priced by default and off with late_mib 0");
+ check(plain.slots == (free_b - reserve_b) / blob, "late_mib 0 is the old sizing");
+ check(a.slots == (free_b - reserve_b - kLateAllocMib * MiB) / blob, "the cache leaves reserve + late allocation");
+ check(plain.slots - a.slots == kLateAllocMib * MiB / blob || plain.slots - a.slots == kLateAllocMib * MiB / blob + 1,
+ "256 MiB fewer bytes: about 100 slots of 2.5 MiB");
+ // the card ends with the reserve: free - slots*blob - priced - 244 MiB unpriced >= reserve
+ check(free_b - a.slots * blob - (184 + 244) * MiB >= 700 * MiB - blob, "the end free is the reserve, not 241 MiB under it");
+ // a small card: not enough for min_slots after the late allocation -> the old sizing (#496 lowers the reserve elsewhere)
+ const AutoCacheSlots small = auto_cache_slots(1200 * MiB, 700 * MiB, blob, min_slots);
+ check(!small.late_priced && small.slots == (500 * MiB) / blob, "a card that cannot spare it keeps the old number");
+ const AutoCacheSlots tiny = auto_cache_slots(900 * MiB, 700 * MiB, blob, min_slots);
+ check(!tiny.late_priced && tiny.slots == 80, "less than min_slots stays as it was");
+ check(auto_cache_slots(500 * MiB, 700 * MiB, blob, min_slots).slots == 0, "a negative room is no slots");
+ check(auto_cache_slots(1000 * MiB, 700 * MiB, 0, min_slots).slots == 0, "no blob size: no slots, no division");
+ // exactly at the limit: min_slots after the late allocation
+ const int64_t edge_free = reserve_b + (kLateAllocMib * MiB) + min_slots * blob;
+ check(auto_cache_slots(edge_free, reserve_b, blob, min_slots).late_priced, "exactly min_slots after the late allocation: priced");
+ check(!auto_cache_slots(edge_free - 1, reserve_b, blob, min_slots).late_priced, "one byte less: the old sizing");
+ std::printf("auto_cache_test: %d checks passed\n", checks);
+ return 0;
+}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 8020d812e..e998d90c5 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -30,6 +30,7 @@
#include "strata/core/conversation_file.hpp"
#include "strata/core/session_save_reclaim.hpp"
#include "strata/core/conversation_memory.hpp"
+#include "strata/core/auto_cache.hpp"
#include "strata/core/ple_residency.hpp"
#include "strata/core/defaults_026.hpp"
#include "strata/core/coupled_draft.hpp"
@@ -6862,7 +6863,17 @@ int main(int argc, char** argv) {
#endif
const int64_t reserve = (((int64_t) o.vram_reserve_mib + prefill_mib) << 20) + mtp_bind + pipe_first;
const int64_t blob = (int64_t) nibble_max_slot_bytes(); // max_blob (STRATA_EXPERT_NIBBLE: or a larger 4-bit slot)
- int64_t slots = ((int64_t) free_b - reserve) / blob;
+ // #496: the least a working cache needs (the verify window cannot start without one, #174): a cache too small to lend the prompt path a
+ // 256-token chunk's buffers (plus the 128 slots a loan leaves; one slot without --prefill) makes it allocate its own on top
+ const int64_t min_slots = (o.prefill_chunk > 0 && pf_borrow)
+ ? ((int64_t) strata::prefill::Prefill::bytes_needed(g, ss, 256) + blob - 1) / blob + 128 : 1;
+ // #4474: what the load allocates after the cache that no setting prices (cuBLAS and the lazily loaded modules the prompt path and the
+ // verifier warm up, ~245 MiB on every card measured) came out of the reserve, so a card ended 240 MiB under the reserve it was asked to
+ // keep (459 MiB free at --vram-reserve-mib 700: under the 512 MiB line the serve check warns about, #2065). The cache now leaves it too.
+ // Not on a card that could not keep min_slots after it, and not under STRATA_DEFAULTS=026 (the v0.2.6 sizing).
+ const strata::core::AutoCacheSlots auto_slots = strata::core::auto_cache_slots(
+ (int64_t) free_b, reserve, blob, min_slots, strata::qwen_defaults_026() ? 0 : strata::core::kLateAllocMib);
+ int64_t slots = auto_slots.slots;
if (!profile.empty()) slots = std::min<int64_t>(slots, (int64_t) profile.size());
#if defined(STRATA_GPU_METAL)
// Metal: the decode window reads its experts from its own arena (metal/src/engine/window.cpp), and the device copies the shared host
@@ -6879,8 +6890,9 @@ int main(int argc, char** argv) {
#endif
o.expert_cache = (int) std::max<int64_t>(slots, 0);
std::fprintf(stderr, "strata generate: expert cache auto: %.2f GiB free, %d MiB reserved (+%lld MiB for the "
- "draft head) -> %d slots\n",
- (double) free_b / 1073741824.0, o.vram_reserve_mib, (long long) (mtp_bind >> 20), o.expert_cache);
+ "draft head%s) -> %d slots\n",
+ (double) free_b / 1073741824.0, o.vram_reserve_mib, (long long) (mtp_bind >> 20),
+ auto_slots.late_priced ? ", +256 MiB for what is allocated after the cache" : "", o.expert_cache);
#if defined(STRATA_USE_HIP) && !defined(_WIN32)
// An APU's "VRAM" is system RAM: the device-free figure above counts the whole GPU-addressable pool and does not
// subtract ordinary CPU allocations (the host expert arena), so a cache sized from it alone can ask the OOM killer
@@ -6912,8 +6924,6 @@ int main(int argc, char** argv) {
// stops, saying what is short and what makes room. A card the default reserve leaves that much is sized as
// before.
constexpr int kSmallReserveMib = 300;
- const int64_t min_slots = (o.prefill_chunk > 0 && pf_borrow)
- ? ((int64_t) strata::prefill::Prefill::bytes_needed(g, ss, 256) + blob - 1) / blob + 128 : 1;
if (o.expert_cache < min_slots && !o.vram_reserve_given && o.vram_reserve_mib > kSmallReserveMib) {
// the largest reserve (in MiB) that still leaves min_slots
const int64_t fit_mib = ((int64_t) free_b - mtp_bind - pipe_first - min_slots * blob) / (1 << 20) - prefill_mib;