Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: the expert share leaves room for other programs' anonymous memory (a 60 GB Qwen server beside the engine froze the M5 Max twice)
Niko1221e3ed743f2109
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 4767969dd..83a97ac58 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4218,7 +4218,24 @@ struct Model::Impl {mach_msg_type_number_t cnt = TASK_VM_INFO_COUNT;double foot = 0;if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &ti, &cnt) == KERN_SUCCESS) foot = (double) ti.phys_footprint;- return std::max(4.0 * gb, cap - foot - 1.0 * gb);+ // Other programs' anonymous memory (a browser, another model's server). The cap above was measured with a small server beside+ // the engine (NEIGHBOUR_ALLOWANCE); with more anonymous memory held elsewhere the same share leaves macOS too little and the Mac+ // pages (a 128 GB M5 Max froze twice with a 60 GB Qwen server running next to this engine). The rest comes off the experts, so+ // they stream from the SSD a little more. STRATA_METAL_NEIGHBOURS=0 turns it off.+ double neighbours = 0;+ const char* ne = std::getenv("STRATA_METAL_NEIGHBOURS");+ if (ne == nullptr || std::atoi(ne) != 0) {+ vm_statistics64_data_t vs;+ mach_msg_type_number_t vc = HOST_VM_INFO64_COUNT;+ if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t) &vs, &vc) == KERN_SUCCESS) {+ const double anon = ((double) vs.internal_page_count + (double) vs.total_uncompressed_pages_in_compressor) * (double) vm_page_size;+ constexpr double NEIGHBOUR_ALLOWANCE = 8.0e9;+ neighbours = std::max(0.0, anon - foot - NEIGHBOUR_ALLOWANCE);+ if (neighbours > 0.5e9)+ std::fprintf(stderr, "ds4: other programs hold %.1f GB of anonymous memory: %.1f GB less for the experts\n", anon / gb, neighbours / gb);+ }+ }+ return std::max(4.0 * gb, cap - foot - 1.0 * gb - neighbours);}#endif#if defined(STRATA_METAL)