Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: automatic VRAM-expert share from the anonymous-memory cap (explicit --vram-experts wins)
Niko122134810a3bb3bf
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex b07b1b9fe..283f43639 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -772,7 +772,7 @@ int main(int argc, char** argv) {for (int g : parse_ids(val("--gpus"))) o.gpus.push_back(g);} else if (a == "--max-context") o.max_context = std::atoll(val("--max-context").c_str());else if (a == "--reserve-mib") o.reserve_mib = std::atoll(val("--reserve-mib").c_str());- else if (a == "--vram-experts") o.vram_experts = std::atof(val("--vram-experts").c_str());+ else if (a == "--vram-experts") { o.vram_experts = std::atof(val("--vram-experts").c_str()); o.vram_experts_auto = false; }else if (a == "--cpu-threads") o.cpu_threads = std::atoi(val("--cpu-threads").c_str());else if (a == "--expert-profile") o.profile = val("--expert-profile");else if (a == "--ssd-cache") o.ssd_cache_gb = std::atof(val("--ssd-cache").c_str());diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex e03c190ba..b458494f6 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -31,6 +31,10 @@#include <cublas_v2.h>#include <cuda_runtime.h>+#if defined(STRATA_METAL)+#include <sys/sysctl.h>+#include <mach/mach.h>+#endif#include <algorithm>#include <array>#include <condition_variable>@@ -3552,6 +3556,36 @@ struct Model::Impl {return true;}+#if defined(STRATA_METAL)+ // The most the VRAM experts may add on a Mac: the engine's anonymous memory (Metal buffers + host memory, all counted) stays+ // under the cap metal/src/engine/arena_budget.h derived from the freezes of the 128 GB M5 Max (macOS keeps a file-cache floor and+ // compresses / swaps anonymous memory past it): max(15% of the RAM, 40% of the RAM, the floor + 12 GB) left to macOS, at most+ // the GPU working set; what the process holds now (task_info phys_footprint) is already inside it.+ double metal_expert_cap() const {+ uint64_t phys = 0;+ size_t len = sizeof phys;+ sysctlbyname("hw.memsize", &phys, &len, nullptr, 0);+ unsigned long long pages = 0;+ size_t pl = sizeof pages;+ double floor_b = 0;+ if (sysctlbyname("vm.vm_page_filecache_min", &pages, &pl, nullptr, 0) == 0) floor_b = (double) pages * (double) vm_page_size;+ size_t fr = 0, tot = 0;+ cudaMemGetInfo(&fr, &tot);+ const double gb = 1e9, P = (double) phys;+ double keep = std::max(7.0 * gb, 0.15 * P);+ if (const char* e = std::getenv("STRATA_METAL_KEEP_FREE_GB")) keep = std::atof(e) * gb;+ double share = 0.40;+ if (const char* e = std::getenv("STRATA_METAL_LEAVE_SHARE")) share = std::atof(e);+ double leave = keep;+ if (P >= 64.0 * gb) leave = std::max({leave, share * P, floor_b > 0 ? floor_b + 12.0 * gb : 0.0});+ const double cap = std::min((double) tot, P - leave);+ task_vm_info_data_t ti;+ mach_msg_type_number_t cnt = TASK_VM_INFO_COUNT;+ double foot = 0;+ if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &ti, &cnt) == KERN_SUCCESS) foot = (double) ti.phys_footprint;+ return std::max(4.0 * gb, cap - foot - 1.0 * gb);+ }+#endif#if defined(STRATA_METAL)// Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it// like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).@@ -3690,6 +3724,16 @@ struct Model::Impl {CK(cudaMemGetInfo(&fr, &tot));budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20))) *o.vram_experts);+#if defined(STRATA_METAL)+ if (o.vram_experts_auto && std::getenv("DS4_NO_AUTO_VX") == nullptr) {+ const double cap = metal_expert_cap();+ if (cap < budget) {+ std::fprintf(stderr, "ds4: expert share: %.1f GB of VRAM experts (the RAM cap leaves %.1f GB; --vram-experts sets it)\n",+ cap / 1e9, cap / 1e9);+ budget = cap;+ }+ }+#endifcharged = charged || commit_charged(devs[(size_t) m]);}if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppindex c291d13ac..f8b087c70 100644--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@ -25,6 +25,7 @@ struct Options {int64_t max_context = 32768;std::vector<int> gpus; // empty: every visible GPUint64_t reserve_mib = 600; // VRAM left free on each GPU+ bool vram_experts_auto = true; // (Metal: no --vram-experts given: the share the anonymous-memory cap leaves - metal_expert_cap)double vram_experts = 1.0; // the share of the expert budget to use (tests: less, so the CPU lane works)int cpu_threads = 0; // 0: the physical coresstd::string profile; // expert hit counts (read to place the hottest experts in VRAM, written back)