Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal: batched q8 quantiser, zero-copy expert mapping (DS4_ZC), spin-wait, GPU-busy and timing switches
Niko12215dce2f63e904
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.hindex 8d7205341..ac298c9db 100644--- a/metal/include/metal_compat/ds4_metal_host.h+++ b/metal/include/metal_compat/ds4_metal_host.h@@ -50,6 +50,8 @@ bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_tbool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out);+uint64_t zc_map(const void* p, size_t bytes); // zero-copy weights: the GPU address of host memory, 0 if not mappable+bool q8_quant_multi(const void* specs, int n); // {const float* x; uint64 ldx; uint32 rows, k; int8* q; float* d} eachbool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td);bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td);bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out);diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 9b29796d1..e78b93811 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -67,6 +67,7 @@ bool ensure() {}if (g_cx) return g_lib >= 0;g_cx = new mtl::Context();+ g_cx->start_keepalive();g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);@@ -524,6 +525,53 @@ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float}+// the GPU address of host weight memory (see mtl::Context::zc_map); 0: not available+uint64_t zc_map(const void* p, size_t bytes) {+ if (!ensure()) return 0;+ return g_cx->zc_map(p, bytes);+}++// Several ds4_q8_quant calls in one dispatch (specs: {x, ldx, rows, k, q, d} with the engine's device pointers, which are the+// kernels' own addresses, as iq_mv's jobs carry them)+namespace {+struct HostQ8Spec {+ const float* x;+ uint64_t ldx;+ uint32_t rows, k;+ int8_t* q;+ float* d;+};+} // namespace+bool q8_quant_multi(const void* specs, int n) {+ if (n <= 0) return true;+ if (!ensure()) return false;+ const HostQ8Spec* sp = (const HostQ8Spec*) specs;+ uint32_t max_rows = 1, max_k = 32;+ for (int i = 0; i < n; ++i) {+ max_rows = std::max(max_rows, sp[i].rows);+ max_k = std::max(max_k, sp[i].k);+ }+ mtl::Buffer bj = g_cx->alloc((size_t) n * sizeof(HostQ8Spec));+ std::memcpy(bj.ptr, sp, (size_t) n * sizeof(HostQ8Spec));+ static mtl::Pipeline pl;+ static bool have = false;+ if (!have) {+ try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant_multi"); } catch (...) { g_cx->free(bj); return false; }+ have = true;+ }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (max_k / 32 + 7) / 8;+ d.grid[1] = max_rows;+ d.grid[2] = (uint32_t) n;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+}++// ds4_gguf.cu:371 - the MXFP4 expert matvec. The engine builds its BfJobs array on the HOST (with device pointers inside), so// the jobs go to the kernel as an opaque constant copy: same bytes, same layout, indexed by threadgroup_position_in_grid.y.namespace {@@ -617,9 +665,15 @@ bool iq_mv(const void* jobs, int nj) {// (ds4_*_mvn<NC>); a job of more columns is cut into pieces of 8. Other types keep the 16-column kernels.std::map<std::string, std::vector<Ds4IqMvJob>> groups;int max_n = 0;+ if (std::getenv("DS4_DBG_JOBS")) {+ std::fprintf(stderr, "iq_mv call:");+ for (int i = 0; i < nj; ++i) std::fprintf(stderr, " [t%d n%d k%d c%d]", src[i].type, src[i].n, src[i].k, src[i].ncol);+ std::fprintf(stderr, "\n");+ }static const bool old_only = std::getenv("DS4_NO_MVN") != nullptr;for (int i = 0; i < nj; ++i) {const Ds4IqMvJob& q = src[i];+ { static const char* sk = std::getenv("DS4_SKIP_TYPES"); if (sk != nullptr && std::strstr(sk, ("," + std::to_string(q.type) + ",").c_str()) != nullptr) continue; } // (timing only)if (q.n > max_n) max_n = q.n;const int ncol = q.ncol > 0 ? q.ncol : 1;const char* tn = nullptr;diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex c9d49e89e..451e15c08 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -2161,3 +2161,27 @@ kernel void ds4_axpby_rows(device const float* t [[buffer(0)]], device float* yconst ulong yi = (ulong) tgp.y * ldy + c;y[yi] = alpha * t[(ulong) tgp.y * n + c] + (beta != 0.0f ? beta * y[yi] : 0.0f);}++// ds4_q8_quant for several jobs in one dispatch (a matvec call's activations): grid (groups, max rows, jobs)+struct Ds4Q8Spec {+ device const float* x;+ ulong ldx;+ uint rows;+ uint k;+ device char* q;+ device float* d;+};+kernel void ds4_q8_quant_multi(device const Ds4Q8Spec* specs [[buffer(0)]], uint3 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const device Ds4Q8Spec& s = specs[tgp.z];+ const uint warp = tid / 32u, lane = tid % 32u;+ const uint b = tgp.x * 8u + warp;+ const uint r = tgp.y;+ if (r >= s.rows || (size_t) b * 32u >= (size_t) s.k) return;+ const device float* xr = s.x + (size_t) r * s.ldx;+ const float v = xr[(size_t) b * 32u + lane];+ const float am = simd_max(fabs(v));+ const float sc = am / 127.0f;+ s.q[(size_t) r * s.k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);+ if (lane == 0u) s.d[(size_t) r * (s.k / 32u) + b] = sc;+}diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.hindex 831d8d3a5..c60a0535d 100644--- a/metal/src/mtl_rt.h+++ b/metal/src/mtl_rt.h@@ -83,6 +83,15 @@ public:bool supports_metal4() const;uint64_t recommended_working_set() const;uint64_t max_buffer_length() const;+ // Keeps the GPU out of its low-clock idle state while dispatches keep coming (a lone kernel in a quiet GPU runs 2-3x slower:+ // measured 205 vs 69 us for the same matvec): a spin kernel of one threadgroup on a second queue, stopped 50 ms after the+ // last dispatch. DS4_NO_KEEPALIVE=1: off.+ void start_keepalive();+ // Zero-copy weights: the GPU address of host memory [p, p + bytes) (file-backed pages of the model, RAM slots) through a+ // no-copy buffer over the pages it spans, made resident. 0: not mappable (the cap on buffers is reached, or the pages are not+ // ours). The first dispatch that reads cold pages waits for them (the buffer is wired on use).+ uint64_t zc_map(const void* p, size_t bytes);+ size_t zc_count() const;uint64_t allocated() const { return allocated_; } // bytes of alloc() + wrap() buffers so far// Compiles a library from MSL source files (paths), prepended by `prelude` (defines) and, if include_ggml,diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 79ed8fe37..93c75be27 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -13,6 +13,8 @@#include <unordered_map>#include <string>#include <deque>+#include <map>+#include <thread>#include <cstdio>#include <cstdlib>#include <fstream>@@ -126,6 +128,90 @@ int Context::compile(const std::vector<std::string>& files, const std::string& preturn (int)impl_->libs.size() - 1;}++// ---- the keep-alive spin (see mtl_rt.h)+static std::atomic<int64_t> g_last_act_ns{0};+static inline int64_t ka_now_ns() { return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); }+static std::atomic<bool> g_ka_on{false};++void Context::start_keepalive() {+ if (!std::getenv("DS4_KEEPALIVE") || g_ka_on.exchange(true)) return; // (measured slower in the decode loop)+ id<MTLDevice> dev = impl_->dev;+ std::thread([dev]() {+ @autoreleasepool {+ static const char* kSpin = R"MSL(+#include <metal_stdlib>+using namespace metal;+kernel void ka_spin(device atomic_uint* c [[buffer(0)]], constant uint& n [[buffer(1)]], uint t [[thread_position_in_grid]]) {+ uint x = t;+ for (uint i = 0; i < n; ++i) x = x * 1664525u + 1013904223u;+ if (x == 0xdeadbeefu) atomic_fetch_add_explicit(c, 1u, memory_order_relaxed);+}+)MSL";+ NSError* err = nil;+ id<MTLLibrary> lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kSpin] options:nil error:&err];+ if (!lib) return;+ id<MTLComputePipelineState> ps = [dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"ka_spin"] error:&err];+ if (!ps) return;+ id<MTLCommandQueue> q = [dev newCommandQueue];+ id<MTLBuffer> cbuf = [dev newBufferWithLength:64 options:MTLResourceStorageModeShared];+ uint32_t iters = std::getenv("DS4_KEEPALIVE_ITERS") ? (uint32_t) std::atoi(std::getenv("DS4_KEEPALIVE_ITERS")) : 20000u;+ for (;;) {+ if (ka_now_ns() - g_last_act_ns.load(std::memory_order_relaxed) > 50000000LL) {+ std::this_thread::sleep_for(std::chrono::milliseconds(2));+ continue;+ }+ @autoreleasepool {+ id<MTLCommandBuffer> cb = [q commandBuffer];+ id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];+ [enc setComputePipelineState:ps];+ [enc setBuffer:cbuf offset:0 atIndex:0];+ [enc setBytes:&iters length:4 atIndex:1];+ [enc dispatchThreads:MTLSizeMake(256, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];+ [enc endEncoding];+ [cb commit];+ [cb waitUntilCompleted];+ }+ }+ }+ }).detach();+}+++// ---- zero-copy weights (see mtl_rt.h)+namespace {+struct ZcBuf { size_t len; void* handle; uint64_t addr; };+std::map<uintptr_t, ZcBuf> g_zc;+std::mutex g_zc_mu;+} // namespace++uint64_t Context::zc_map(const void* p, size_t bytes) {+ const size_t pg = 16384;+ const uintptr_t lo = (uintptr_t) p & ~(uintptr_t) (pg - 1);+ const uintptr_t hi = ((uintptr_t) p + bytes + pg - 1) & ~(uintptr_t) (pg - 1);+ std::lock_guard<std::mutex> lk(g_zc_mu);+ auto it = g_zc.upper_bound(lo);+ if (it != g_zc.begin()) {+ --it;+ if (it->first <= lo && hi <= it->first + it->second.len) return it->second.addr + ((uintptr_t) p - it->first);+ }+ static const size_t cap = std::getenv("DS4_ZC_MAX") ? (size_t) std::atoll(std::getenv("DS4_ZC_MAX")) : 9000;+ if (g_zc.size() >= cap) return 0;+ id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:(void*) lo length:hi - lo options:MTLResourceStorageModeShared deallocator:nil];+ if (!b) return 0;+ if (ds4_gpu_residency_set) {+ id<MTLResidencySet> rs = (__bridge id<MTLResidencySet>)ds4_gpu_residency_set();+ if (rs) {+ [rs addAllocation:b];+ [rs commit];+ }+ }+ ZcBuf z{hi - lo, (__bridge_retained void*) b, (uint64_t) [b gpuAddress]};+ g_zc[lo] = z;+ return z.addr + ((uintptr_t) p - lo);+}+size_t Context::zc_count() const { std::lock_guard<std::mutex> lk(g_zc_mu); return g_zc.size(); }+Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {NSError* err = nil;id<MTLFunction> f;@@ -324,7 +410,12 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {static std::deque<std::string> g_recent; // the last dispatches encoded (printed with a command buffer failure)static std::string recent_str() { std::string r; for (auto& n : g_recent) r += n + " "; return r; }-static double g_gpu_ms = 0.0; // GPU time of every committed command buffer, summed+static double g_gpu_ms = 0.0;+static std::atomic<uint64_t> g_gpu_busy_us{0}; // DS4_GPU_TOTAL: the GPU time of every committed command buffer+static void add_busy_handler(id<MTLCommandBuffer> cb) {+ static const bool on = std::getenv("DS4_GPU_TOTAL") != nullptr;+ if (on) [cb addCompletedHandler:^(id<MTLCommandBuffer> c) { const double d = c.GPUEndTime - c.GPUStartTime; if (d > 0) g_gpu_busy_us += (uint64_t) (d * 1e6); }];+} // GPU time of every committed command buffer, summedstatic std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host time: waiting on command buffers; encoding dispatchesstatic std::mutex g_prof_mu;static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)@@ -333,6 +424,7 @@ static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded, and commastatic std::atomic<uint64_t> g_cb_count{0};static bool g_count_reg = false;static void ds4_report_counts() {+ if (g_gpu_busy_us.load()) std::fprintf(stderr, "mtl: GPU busy (every command buffer) %.1f ms\n", g_gpu_busy_us.load() / 1000.0);std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,(unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());std::fprintf(stderr, "mtl: host waited %.1f ms for the GPU in %llu waits; encoding the dispatches took %.1f ms\n", g_wait_ns.load() / 1e6, (unsigned long long) g_wait_n.load(), g_enc_ns.load() / 1e6);@@ -370,6 +462,7 @@ void Context::split_runs() {[impl_->enc_def endEncoding];if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];[impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];+ add_busy_handler(impl_->cb_def);[impl_->cb_def commit];impl_->inflight = impl_->cb_def;impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();@@ -378,6 +471,20 @@ void Context::split_runs() {impl_->def_n = 0;}+// Waiting for a committed command buffer: spin on its status for a while (the thread wake of waitUntilCompleted costs+// ~1 ms a wait; the decode token waits ~100 times), then block. DS4_NO_SPIN=1: always block.+static void wait_spin(id<MTLCommandBuffer> cb) {+ static const bool no_spin = std::getenv("DS4_NO_SPIN") != nullptr;+ static const double budget = std::getenv("DS4_SPIN_US") ? std::atof(std::getenv("DS4_SPIN_US")) * 1e-6 : 20e-3;+ if (!no_spin) {+ const auto t0 = std::chrono::steady_clock::now();+ while (cb.status < MTLCommandBufferStatusCompleted) {+ if (std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() > budget) break;+ }+ }+ if (cb.status < MTLCommandBufferStatusCompleted) [cb waitUntilCompleted];+}+void* Context::mark() {std::lock_guard<std::recursive_mutex> lk_(impl_->mu);if (impl_->cb_def != nil) split_runs(); // (the streams' pending work goes first, inside)@@ -388,7 +495,7 @@ void* Context::mark() {void Context::wait_mark(void* token) {id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;const auto tw0 = std::chrono::steady_clock::now();- [cb waitUntilCompleted];+ wait_spin(cb);g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();++g_wait_n;if (cb.status != MTLCommandBufferStatusCompleted)@@ -405,6 +512,7 @@ void Context::flush_runs() {[impl_->enc_def endEncoding];if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];[impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];+ add_busy_handler(impl_->cb_def);[impl_->cb_def commit];impl_->inflight = impl_->cb_def;impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();@@ -415,7 +523,7 @@ void Context::flush_runs() {if (impl_->inflight != nil) {if (std::getenv("DS4_TRACE_WAIT")) { void* fr[8]; const int nf = backtrace(fr, 8); std::fprintf(stderr, "WAIT flush_runs:\n"); backtrace_symbols_fd(fr + 1, nf - 1, 2); }const auto tw0 = std::chrono::steady_clock::now();- [impl_->inflight waitUntilCompleted];+ wait_spin(impl_->inflight);g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();++g_wait_n;{ const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;@@ -447,6 +555,7 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,}const auto te0 = std::chrono::steady_clock::now();encode(impl_->enc_def, ds[0]);+ g_last_act_ns.store(ka_now_ns(), std::memory_order_relaxed);g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 297025f51..ba4ed3e3d 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1093,6 +1093,15 @@ x3::Mat view(const x3::Mat& m, int c0, int nc) {bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {if (n <= 0 || n > x3::kMaxMvJobs) return false;+#if defined(STRATA_METAL)+ // the jobs' int8 activations are quantised in one dispatch before the matvecs (a dispatch a job was 45% of the decode token's)+ struct Q8Batch { const float* x; uint64_t ldx; uint32_t rows, k; int8_t* q; float* d; } q8_batch[x3::kMaxMvJobs];+ int nq8 = 0;+ const bool q8_deferred = std::getenv("DS4_NO_Q8_BATCH") == nullptr && std::getenv("DS4_DBG_Q8") == nullptr;+#define Q8_QUANT(XF, LDX, ROWS, KK, Q, D, S) do { if (q8_deferred) q8_batch[nq8++] = Q8Batch{XF, (uint64_t) (LDX), (uint32_t) (ROWS), (uint32_t) (KK), Q, D}; else q8_quant(XF, LDX, ROWS, KK, Q, D, S); } while (0)+#else+#define Q8_QUANT(XF, LDX, ROWS, KK, Q, D, S) q8_quant(XF, LDX, ROWS, KK, Q, D, S)+#endifBfJobs J[3] = {}; // the BF16 jobs, the MXFP4 ones, the Q8_0 ones: a launch eachint nj[3] = {0, 0, 0}, max_n[3] = {0, 0, 0}, max_c[3] = {1, 1, 1};bool vec[3] = {true, true, true};@@ -1113,7 +1122,7 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {return false;}const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;- q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);+ Q8_QUANT(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);iq::MvJob& b = IJ[nij++];b.w = (const uint8_t*) m.raw;b.ld = row_stride(m);@@ -1165,7 +1174,7 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {#if defined(STRATA_METAL)if (std::getenv("DS4_DBG_Q8_PRE")) ds4_metal_flush(); // everything before the quantiser has finished#endif- q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);+ Q8_QUANT(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);#if defined(STRATA_METAL)if (std::getenv("DS4_DBG_Q8")) { // DBG: the quantiser against a host recomputeds4_metal_flush();@@ -1197,6 +1206,9 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));}}+#if defined(STRATA_METAL)+ if (nq8 > 0) ::strata::dsv41::metal_host::q8_quant_multi(q8_batch, nq8);+#endiffor (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);#if defined(STRATA_METAL)if (std::getenv("DS4_DBG_Q8") && nj[2] > 0) { // DBG: the Q8_0 matvec's first job against a host recomputediff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex bbd90ef1a..e03c190ba 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -3552,6 +3552,30 @@ struct Model::Impl {return true;}+#if defined(STRATA_METAL)+ // Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it+ // like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).+ bool zc_try(Layer& l, int e) {+ static const bool on = std::getenv("DS4_ZC") != nullptr; // (measured: no faster than the CPU lane - cold pages come from the SSD either way)+ if (!on || l.lay.fmt != gg::kFmtMXFP4 || l.nm != 1 || l.vm.empty()) return false;+ if (l.vm[(size_t) e][0].raw != nullptr) return true;+ uint64_t a[3];+ for (int r = 0; r < 3; ++r) {+ a[r] = ::strata::dsv41::metal_host::zc_map(mx_mat(l, e, r), mx_part_bytes(l, r));+ if (a[r] == 0) return false;+ }+ for (int r = 0; r < 3; ++r) {+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];+ m = x3::Mat{};+ m.K = gg::expert_type(l.lay, r);+ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;+ m.raw = (const void*) a[r];+ m.k = r < 2 ? g.dim : g.ff;+ m.n = r < 2 ? g.ff : g.dim;+ }+ return true;+ }+#endifvoid set_vm(Layer& l, int e, int slot) {uint8_t* b = l.vram_base + (size_t) slot * l.blob;if (l.lay.fmt == gg::kFmtMXFP4) {@@ -8256,7 +8280,12 @@ struct Model::Impl {for (int e = 0; e < l.n_exp; ++e) {const auto& pe = per[(size_t) e];if (pe.empty()) continue;- if (l.slot[(size_t) e] >= 0) {+#if defined(STRATA_METAL)+ const bool on_gpu = l.slot[(size_t) e] >= 0 || (T == 1 && zc_try(l, e));+#else+ const bool on_gpu = l.slot[(size_t) e] >= 0;+#endif+ if (on_gpu) {vex[nvex] = e;vrow0[nvex] = nrows;vn[nvex] = (int) pe.size();@@ -8316,7 +8345,8 @@ struct Model::Impl {#if defined(STRATA_METAL)ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)#endif- cpu_experts(l, h_x, ents);+ static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)+ if (!skip_cpu) cpu_experts(l, h_x, ents);std::fill(h_part, h_part + (size_t) T * D, 0.0f);for (int sl = 0; sl < nslots; ++sl) {float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;