Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Qwen: the automatic expert cache leaves the reserve it was asked to keep - the ~245 MiB the load allocates after it are priced (#4474)
Niko1221cfa197e21a32
diff --git a/CMakeLists.txt b/CMakeLists.txtindex ee0599211..6d1c9a858 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1633,6 +1633,11 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_include_directories(ple_residency_test PRIVATE include)add_test(NAME ple_residency_test COMMAND ple_residency_test)+ # #4474: the automatic expert cache's slot rule (the late allocation priced); header-only, needs nothing+ add_executable(auto_cache_test src/core/auto_cache_test.cpp)+ target_include_directories(auto_cache_test PRIVATE include)+ add_test(NAME auto_cache_test COMMAND auto_cache_test)+# 0.2.7 (#3911): the qwen35 host sampler's penalties on a short (draft-vocab) logit row; header-only, CPUadd_executable(qwen35_sampling_test src/qwen35/sampling_test.cpp)target_include_directories(qwen35_sampling_test PRIVATE include)diff --git a/include/strata/core/auto_cache.hpp b/include/strata/core/auto_cache.hppnew file mode 100644index 000000000..9b7249e81--- /dev/null+++ b/include/strata/core/auto_cache.hpp@@ -0,0 +1,47 @@+// include/strata/core/auto_cache.hpp - the slot count of the automatic expert cache (--expert-cache auto), as a rule with a test.+//+// The auto cache is sized from the VRAM free after the weights, the session and the drafter, minus --vram-reserve-mib (the reserve is what+// the operator wants LEFT once everything is loaded), the draft head that binds later (priced), the prompt path's own buffers when it does+// not borrow cache slots (priced) and the second verifier of --pipeline-windows (priced). What was never priced is what the load allocates+// AFTER the cache whatever the settings: the CUDA libraries the prompt path and the verifier warm up (cuBLAS handles and workspaces, the+// lazily loaded modules) and the verifier's small buffers. Measured with STRATA_TRACE=1 on the same build, config and flags+// (--vram-reserve-mib 700, 256K context, MTP, 4 drafts):+// RTX A4000 16 GB (z440, IQ3_XXS): 883 MiB free after the cache -> 719 after the head and the prompt path (-164) -> 461 after the+// verifier and the drafter's binding (-258 = 178 MiB draft head, priced, + 80) = 244 MiB unpriced+// RTX A6000 48 GB (boston, IQ3_S): 919 -> 755 (-164) -> 459 (-296 = 213 MiB draft head, priced, + 83) = 247 MiB unpriced+// --prefill 512 / 2048 / auto on the A4000: the same 164 + 80. v0.2.6 ends at 461 MiB on the same card: not new, but every 16 GB card with+// the default 700 MiB ends under the 512 MiB line the engine itself warns about (#2065, #4474), and the warning cannot be cured by the+// operator raising the reserve by the advice's figure without losing the cache it was meant to keep.+// So the reserve is now honest: the cache leaves `reserve + kLateAllocMib`, and the card ends with about the reserve.+#pragma once++#include <cstdint>++namespace strata::core {++/// MiB the load allocates after the cache that no setting prices (see above; measured 244-247, a little margin).+constexpr int64_t kLateAllocMib = 256;++struct AutoCacheSlots {+ int64_t slots = 0; ///< the cache's slot count (of the largest blob), >= 0+ bool late_priced = false; ///< kLateAllocMib was left out of the cache+};++/// `free_b`: VRAM free now; `reserve_b`: the reserve and everything else that is priced (bytes); `blob`: the largest slot; `min_slots`: the least a working+/// cache needs (#496). `late_mib` 0 turns the late allocation off (STRATA_DEFAULTS=026). The late allocation is only left out when the cache still has+/// `min_slots` after it: a small card keeps the sizing it had (its reserve is lowered further elsewhere, #496), never a smaller cache than it can work with.+inline AutoCacheSlots auto_cache_slots(int64_t free_b, int64_t reserve_b, int64_t blob, int64_t min_slots, int64_t late_mib = kLateAllocMib) {+ AutoCacheSlots r;+ if (blob <= 0) return r;+ const int64_t plain = (free_b - reserve_b) / blob;+ const int64_t late = late_mib > 0 ? (free_b - reserve_b - (late_mib << 20)) / blob : plain;+ if (late_mib > 0 && late >= min_slots && late > 0) {+ r.slots = late;+ r.late_priced = true;+ } else {+ r.slots = plain > 0 ? plain : 0;+ }+ return r;+}++} // namespace strata::corediff --git a/src/core/auto_cache_test.cpp b/src/core/auto_cache_test.cppnew file mode 100644index 000000000..c9ee295f2--- /dev/null+++ b/src/core/auto_cache_test.cpp@@ -0,0 +1,43 @@+#include "strata/core/auto_cache.hpp"++#include <cstdio>+#include <cstdlib>++using namespace strata::core;+namespace {+int checks = 0;+void check(bool ok, const char* label) {+ ++checks;+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }+}+constexpr int64_t MiB = 1ll << 20;+}++int main() {+ const int64_t blob = 2560ll * 1024; // 2.5 MiB+ const int64_t min_slots = 300;+ // the A4000 of #4474: 9951 MiB free after the weights, 700 reserved + 184 draft head+ const int64_t free_b = 9951 * MiB, reserve_b = (700 + 184) * MiB;+ const AutoCacheSlots a = auto_cache_slots(free_b, reserve_b, blob, min_slots);+ const AutoCacheSlots plain = auto_cache_slots(free_b, reserve_b, blob, min_slots, 0);+ check(a.late_priced && !plain.late_priced, "the late allocation is priced by default and off with late_mib 0");+ check(plain.slots == (free_b - reserve_b) / blob, "late_mib 0 is the old sizing");+ check(a.slots == (free_b - reserve_b - kLateAllocMib * MiB) / blob, "the cache leaves reserve + late allocation");+ check(plain.slots - a.slots == kLateAllocMib * MiB / blob || plain.slots - a.slots == kLateAllocMib * MiB / blob + 1,+ "256 MiB fewer bytes: about 100 slots of 2.5 MiB");+ // the card ends with the reserve: free - slots*blob - priced - 244 MiB unpriced >= reserve+ check(free_b - a.slots * blob - (184 + 244) * MiB >= 700 * MiB - blob, "the end free is the reserve, not 241 MiB under it");+ // a small card: not enough for min_slots after the late allocation -> the old sizing (#496 lowers the reserve elsewhere)+ const AutoCacheSlots small = auto_cache_slots(1200 * MiB, 700 * MiB, blob, min_slots);+ check(!small.late_priced && small.slots == (500 * MiB) / blob, "a card that cannot spare it keeps the old number");+ const AutoCacheSlots tiny = auto_cache_slots(900 * MiB, 700 * MiB, blob, min_slots);+ check(!tiny.late_priced && tiny.slots == 80, "less than min_slots stays as it was");+ check(auto_cache_slots(500 * MiB, 700 * MiB, blob, min_slots).slots == 0, "a negative room is no slots");+ check(auto_cache_slots(1000 * MiB, 700 * MiB, 0, min_slots).slots == 0, "no blob size: no slots, no division");+ // exactly at the limit: min_slots after the late allocation+ const int64_t edge_free = reserve_b + (kLateAllocMib * MiB) + min_slots * blob;+ check(auto_cache_slots(edge_free, reserve_b, blob, min_slots).late_priced, "exactly min_slots after the late allocation: priced");+ check(!auto_cache_slots(edge_free - 1, reserve_b, blob, min_slots).late_priced, "one byte less: the old sizing");+ std::printf("auto_cache_test: %d checks passed\n", checks);+ return 0;+}diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 8020d812e..e998d90c5 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -30,6 +30,7 @@#include "strata/core/conversation_file.hpp"#include "strata/core/session_save_reclaim.hpp"#include "strata/core/conversation_memory.hpp"+#include "strata/core/auto_cache.hpp"#include "strata/core/ple_residency.hpp"#include "strata/core/defaults_026.hpp"#include "strata/core/coupled_draft.hpp"@@ -6862,7 +6863,17 @@ int main(int argc, char** argv) {#endifconst int64_t reserve = (((int64_t) o.vram_reserve_mib + prefill_mib) << 20) + mtp_bind + pipe_first;const int64_t blob = (int64_t) nibble_max_slot_bytes(); // max_blob (STRATA_EXPERT_NIBBLE: or a larger 4-bit slot)- int64_t slots = ((int64_t) free_b - reserve) / blob;+ // #496: the least a working cache needs (the verify window cannot start without one, #174): a cache too small to lend the prompt path a+ // 256-token chunk's buffers (plus the 128 slots a loan leaves; one slot without --prefill) makes it allocate its own on top+ const int64_t min_slots = (o.prefill_chunk > 0 && pf_borrow)+ ? ((int64_t) strata::prefill::Prefill::bytes_needed(g, ss, 256) + blob - 1) / blob + 128 : 1;+ // #4474: what the load allocates after the cache that no setting prices (cuBLAS and the lazily loaded modules the prompt path and the+ // verifier warm up, ~245 MiB on every card measured) came out of the reserve, so a card ended 240 MiB under the reserve it was asked to+ // keep (459 MiB free at --vram-reserve-mib 700: under the 512 MiB line the serve check warns about, #2065). The cache now leaves it too.+ // Not on a card that could not keep min_slots after it, and not under STRATA_DEFAULTS=026 (the v0.2.6 sizing).+ const strata::core::AutoCacheSlots auto_slots = strata::core::auto_cache_slots(+ (int64_t) free_b, reserve, blob, min_slots, strata::qwen_defaults_026() ? 0 : strata::core::kLateAllocMib);+ int64_t slots = auto_slots.slots;if (!profile.empty()) slots = std::min<int64_t>(slots, (int64_t) profile.size());#if defined(STRATA_GPU_METAL)// Metal: the decode window reads its experts from its own arena (metal/src/engine/window.cpp), and the device copies the shared host@@ -6879,8 +6890,9 @@ int main(int argc, char** argv) {#endifo.expert_cache = (int) std::max<int64_t>(slots, 0);std::fprintf(stderr, "strata generate: expert cache auto: %.2f GiB free, %d MiB reserved (+%lld MiB for the "- "draft head) -> %d slots\n",- (double) free_b / 1073741824.0, o.vram_reserve_mib, (long long) (mtp_bind >> 20), o.expert_cache);+ "draft head%s) -> %d slots\n",+ (double) free_b / 1073741824.0, o.vram_reserve_mib, (long long) (mtp_bind >> 20),+ auto_slots.late_priced ? ", +256 MiB for what is allocated after the cache" : "", o.expert_cache);#if defined(STRATA_USE_HIP) && !defined(_WIN32)// An APU's "VRAM" is system RAM: the device-free figure above counts the whole GPU-addressable pool and does not// subtract ordinary CPU allocations (the host expert arena), so a cache sized from it alone can ask the OOM killer@@ -6912,8 +6924,6 @@ int main(int argc, char** argv) {// stops, saying what is short and what makes room. A card the default reserve leaves that much is sized as// before.constexpr int kSmallReserveMib = 300;- const int64_t min_slots = (o.prefill_chunk > 0 && pf_borrow)- ? ((int64_t) strata::prefill::Prefill::bytes_needed(g, ss, 256) + blob - 1) / blob + 128 : 1;if (o.expert_cache < min_slots && !o.vram_reserve_given && o.vram_reserve_mib > kSmallReserveMib) {// the largest reserve (in MiB) that still leaves min_slotsconst int64_t fit_mib = ((int64_t) free_b - mtp_bind - pipe_first - min_slots * blob) / (1 << 20) - prefill_mib;