Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek Metal: batched q8 quantiser, zero-copy expert mapping (DS4_ZC), spin-wait, GPU-busy and timing switches

Niko12215dce2f63e904
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
index 8d7205341..ac298c9db 100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -50,6 +50,8 @@ bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t
bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);
bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);
bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out);
+uint64_t zc_map(const void* p, size_t bytes); // zero-copy weights: the GPU address of host memory, 0 if not mappable
+bool q8_quant_multi(const void* specs, int n); // {const float* x; uint64 ldx; uint32 rows, k; int8* q; float* d} each
bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td);
bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td);
bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out);
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 9b29796d1..e78b93811 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -67,6 +67,7 @@ bool ensure() {
}
if (g_cx) return g_lib >= 0;
g_cx = new mtl::Context();
+ g_cx->start_keepalive();
g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);
if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");
g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);
@@ -524,6 +525,53 @@ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float
}
+// the GPU address of host weight memory (see mtl::Context::zc_map); 0: not available
+uint64_t zc_map(const void* p, size_t bytes) {
+ if (!ensure()) return 0;
+ return g_cx->zc_map(p, bytes);
+}
+
+// Several ds4_q8_quant calls in one dispatch (specs: {x, ldx, rows, k, q, d} with the engine's device pointers, which are the
+// kernels' own addresses, as iq_mv's jobs carry them)
+namespace {
+struct HostQ8Spec {
+ const float* x;
+ uint64_t ldx;
+ uint32_t rows, k;
+ int8_t* q;
+ float* d;
+};
+} // namespace
+bool q8_quant_multi(const void* specs, int n) {
+ if (n <= 0) return true;
+ if (!ensure()) return false;
+ const HostQ8Spec* sp = (const HostQ8Spec*) specs;
+ uint32_t max_rows = 1, max_k = 32;
+ for (int i = 0; i < n; ++i) {
+ max_rows = std::max(max_rows, sp[i].rows);
+ max_k = std::max(max_k, sp[i].k);
+ }
+ mtl::Buffer bj = g_cx->alloc((size_t) n * sizeof(HostQ8Spec));
+ std::memcpy(bj.ptr, sp, (size_t) n * sizeof(HostQ8Spec));
+ static mtl::Pipeline pl;
+ static bool have = false;
+ if (!have) {
+ try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant_multi"); } catch (...) { g_cx->free(bj); return false; }
+ have = true;
+ }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bj)};
+ d.grid[0] = (max_k / 32 + 7) / 8;
+ d.grid[1] = max_rows;
+ d.grid[2] = (uint32_t) n;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->free(bj);
+ return true;
+}
+
+
// ds4_gguf.cu:371 - the MXFP4 expert matvec. The engine builds its BfJobs array on the HOST (with device pointers inside), so
// the jobs go to the kernel as an opaque constant copy: same bytes, same layout, indexed by threadgroup_position_in_grid.y.
namespace {
@@ -617,9 +665,15 @@ bool iq_mv(const void* jobs, int nj) {
// (ds4_*_mvn<NC>); a job of more columns is cut into pieces of 8. Other types keep the 16-column kernels.
std::map<std::string, std::vector<Ds4IqMvJob>> groups;
int max_n = 0;
+ if (std::getenv("DS4_DBG_JOBS")) {
+ std::fprintf(stderr, "iq_mv call:");
+ for (int i = 0; i < nj; ++i) std::fprintf(stderr, " [t%d n%d k%d c%d]", src[i].type, src[i].n, src[i].k, src[i].ncol);
+ std::fprintf(stderr, "\n");
+ }
static const bool old_only = std::getenv("DS4_NO_MVN") != nullptr;
for (int i = 0; i < nj; ++i) {
const Ds4IqMvJob& q = src[i];
+ { static const char* sk = std::getenv("DS4_SKIP_TYPES"); if (sk != nullptr && std::strstr(sk, ("," + std::to_string(q.type) + ",").c_str()) != nullptr) continue; } // (timing only)
if (q.n > max_n) max_n = q.n;
const int ncol = q.ncol > 0 ? q.ncol : 1;
const char* tn = nullptr;
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index c9d49e89e..451e15c08 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -2161,3 +2161,27 @@ kernel void ds4_axpby_rows(device const float* t [[buffer(0)]], device float* y
const ulong yi = (ulong) tgp.y * ldy + c;
y[yi] = alpha * t[(ulong) tgp.y * n + c] + (beta != 0.0f ? beta * y[yi] : 0.0f);
}
+
+// ds4_q8_quant for several jobs in one dispatch (a matvec call's activations): grid (groups, max rows, jobs)
+struct Ds4Q8Spec {
+ device const float* x;
+ ulong ldx;
+ uint rows;
+ uint k;
+ device char* q;
+ device float* d;
+};
+kernel void ds4_q8_quant_multi(device const Ds4Q8Spec* specs [[buffer(0)]], uint3 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ const device Ds4Q8Spec& s = specs[tgp.z];
+ const uint warp = tid / 32u, lane = tid % 32u;
+ const uint b = tgp.x * 8u + warp;
+ const uint r = tgp.y;
+ if (r >= s.rows || (size_t) b * 32u >= (size_t) s.k) return;
+ const device float* xr = s.x + (size_t) r * s.ldx;
+ const float v = xr[(size_t) b * 32u + lane];
+ const float am = simd_max(fabs(v));
+ const float sc = am / 127.0f;
+ s.q[(size_t) r * s.k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);
+ if (lane == 0u) s.d[(size_t) r * (s.k / 32u) + b] = sc;
+}
diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.h
index 831d8d3a5..c60a0535d 100644
--- a/metal/src/mtl_rt.h
+++ b/metal/src/mtl_rt.h
@@ -83,6 +83,15 @@ public:
bool supports_metal4() const;
uint64_t recommended_working_set() const;
uint64_t max_buffer_length() const;
+ // Keeps the GPU out of its low-clock idle state while dispatches keep coming (a lone kernel in a quiet GPU runs 2-3x slower:
+ // measured 205 vs 69 us for the same matvec): a spin kernel of one threadgroup on a second queue, stopped 50 ms after the
+ // last dispatch. DS4_NO_KEEPALIVE=1: off.
+ void start_keepalive();
+ // Zero-copy weights: the GPU address of host memory [p, p + bytes) (file-backed pages of the model, RAM slots) through a
+ // no-copy buffer over the pages it spans, made resident. 0: not mappable (the cap on buffers is reached, or the pages are not
+ // ours). The first dispatch that reads cold pages waits for them (the buffer is wired on use).
+ uint64_t zc_map(const void* p, size_t bytes);
+ size_t zc_count() const;
uint64_t allocated() const { return allocated_; } // bytes of alloc() + wrap() buffers so far
// Compiles a library from MSL source files (paths), prepended by `prelude` (defines) and, if include_ggml,
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index 79ed8fe37..93c75be27 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -13,6 +13,8 @@
#include <unordered_map>
#include <string>
#include <deque>
+#include <map>
+#include <thread>
#include <cstdio>
#include <cstdlib>
#include <fstream>
@@ -126,6 +128,90 @@ int Context::compile(const std::vector<std::string>& files, const std::string& p
return (int)impl_->libs.size() - 1;
}
+
+// ---- the keep-alive spin (see mtl_rt.h)
+static std::atomic<int64_t> g_last_act_ns{0};
+static inline int64_t ka_now_ns() { return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
+static std::atomic<bool> g_ka_on{false};
+
+void Context::start_keepalive() {
+ if (!std::getenv("DS4_KEEPALIVE") || g_ka_on.exchange(true)) return; // (measured slower in the decode loop)
+ id<MTLDevice> dev = impl_->dev;
+ std::thread([dev]() {
+ @autoreleasepool {
+ static const char* kSpin = R"MSL(
+#include <metal_stdlib>
+using namespace metal;
+kernel void ka_spin(device atomic_uint* c [[buffer(0)]], constant uint& n [[buffer(1)]], uint t [[thread_position_in_grid]]) {
+ uint x = t;
+ for (uint i = 0; i < n; ++i) x = x * 1664525u + 1013904223u;
+ if (x == 0xdeadbeefu) atomic_fetch_add_explicit(c, 1u, memory_order_relaxed);
+}
+)MSL";
+ NSError* err = nil;
+ id<MTLLibrary> lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kSpin] options:nil error:&err];
+ if (!lib) return;
+ id<MTLComputePipelineState> ps = [dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"ka_spin"] error:&err];
+ if (!ps) return;
+ id<MTLCommandQueue> q = [dev newCommandQueue];
+ id<MTLBuffer> cbuf = [dev newBufferWithLength:64 options:MTLResourceStorageModeShared];
+ uint32_t iters = std::getenv("DS4_KEEPALIVE_ITERS") ? (uint32_t) std::atoi(std::getenv("DS4_KEEPALIVE_ITERS")) : 20000u;
+ for (;;) {
+ if (ka_now_ns() - g_last_act_ns.load(std::memory_order_relaxed) > 50000000LL) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(2));
+ continue;
+ }
+ @autoreleasepool {
+ id<MTLCommandBuffer> cb = [q commandBuffer];
+ id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];
+ [enc setComputePipelineState:ps];
+ [enc setBuffer:cbuf offset:0 atIndex:0];
+ [enc setBytes:&iters length:4 atIndex:1];
+ [enc dispatchThreads:MTLSizeMake(256, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
+ [enc endEncoding];
+ [cb commit];
+ [cb waitUntilCompleted];
+ }
+ }
+ }
+ }).detach();
+}
+
+
+// ---- zero-copy weights (see mtl_rt.h)
+namespace {
+struct ZcBuf { size_t len; void* handle; uint64_t addr; };
+std::map<uintptr_t, ZcBuf> g_zc;
+std::mutex g_zc_mu;
+} // namespace
+
+uint64_t Context::zc_map(const void* p, size_t bytes) {
+ const size_t pg = 16384;
+ const uintptr_t lo = (uintptr_t) p & ~(uintptr_t) (pg - 1);
+ const uintptr_t hi = ((uintptr_t) p + bytes + pg - 1) & ~(uintptr_t) (pg - 1);
+ std::lock_guard<std::mutex> lk(g_zc_mu);
+ auto it = g_zc.upper_bound(lo);
+ if (it != g_zc.begin()) {
+ --it;
+ if (it->first <= lo && hi <= it->first + it->second.len) return it->second.addr + ((uintptr_t) p - it->first);
+ }
+ static const size_t cap = std::getenv("DS4_ZC_MAX") ? (size_t) std::atoll(std::getenv("DS4_ZC_MAX")) : 9000;
+ if (g_zc.size() >= cap) return 0;
+ id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:(void*) lo length:hi - lo options:MTLResourceStorageModeShared deallocator:nil];
+ if (!b) return 0;
+ if (ds4_gpu_residency_set) {
+ id<MTLResidencySet> rs = (__bridge id<MTLResidencySet>)ds4_gpu_residency_set();
+ if (rs) {
+ [rs addAllocation:b];
+ [rs commit];
+ }
+ }
+ ZcBuf z{hi - lo, (__bridge_retained void*) b, (uint64_t) [b gpuAddress]};
+ g_zc[lo] = z;
+ return z.addr + ((uintptr_t) p - lo);
+}
+size_t Context::zc_count() const { std::lock_guard<std::mutex> lk(g_zc_mu); return g_zc.size(); }
+
Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {
NSError* err = nil;
id<MTLFunction> f;
@@ -324,7 +410,12 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
static std::deque<std::string> g_recent; // the last dispatches encoded (printed with a command buffer failure)
static std::string recent_str() { std::string r; for (auto& n : g_recent) r += n + " "; return r; }
-static double g_gpu_ms = 0.0; // GPU time of every committed command buffer, summed
+static double g_gpu_ms = 0.0;
+static std::atomic<uint64_t> g_gpu_busy_us{0}; // DS4_GPU_TOTAL: the GPU time of every committed command buffer
+static void add_busy_handler(id<MTLCommandBuffer> cb) {
+ static const bool on = std::getenv("DS4_GPU_TOTAL") != nullptr;
+ if (on) [cb addCompletedHandler:^(id<MTLCommandBuffer> c) { const double d = c.GPUEndTime - c.GPUStartTime; if (d > 0) g_gpu_busy_us += (uint64_t) (d * 1e6); }];
+} // GPU time of every committed command buffer, summed
static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host time: waiting on command buffers; encoding dispatches
static std::mutex g_prof_mu;
static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)
@@ -333,6 +424,7 @@ static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded, and comma
static std::atomic<uint64_t> g_cb_count{0};
static bool g_count_reg = false;
static void ds4_report_counts() {
+ if (g_gpu_busy_us.load()) std::fprintf(stderr, "mtl: GPU busy (every command buffer) %.1f ms\n", g_gpu_busy_us.load() / 1000.0);
std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,
(unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());
std::fprintf(stderr, "mtl: host waited %.1f ms for the GPU in %llu waits; encoding the dispatches took %.1f ms\n", g_wait_ns.load() / 1e6, (unsigned long long) g_wait_n.load(), g_enc_ns.load() / 1e6);
@@ -370,6 +462,7 @@ void Context::split_runs() {
[impl_->enc_def endEncoding];
if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];
[impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
+ add_busy_handler(impl_->cb_def);
[impl_->cb_def commit];
impl_->inflight = impl_->cb_def;
impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();
@@ -378,6 +471,20 @@ void Context::split_runs() {
impl_->def_n = 0;
}
+// Waiting for a committed command buffer: spin on its status for a while (the thread wake of waitUntilCompleted costs
+// ~1 ms a wait; the decode token waits ~100 times), then block. DS4_NO_SPIN=1: always block.
+static void wait_spin(id<MTLCommandBuffer> cb) {
+ static const bool no_spin = std::getenv("DS4_NO_SPIN") != nullptr;
+ static const double budget = std::getenv("DS4_SPIN_US") ? std::atof(std::getenv("DS4_SPIN_US")) * 1e-6 : 20e-3;
+ if (!no_spin) {
+ const auto t0 = std::chrono::steady_clock::now();
+ while (cb.status < MTLCommandBufferStatusCompleted) {
+ if (std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() > budget) break;
+ }
+ }
+ if (cb.status < MTLCommandBufferStatusCompleted) [cb waitUntilCompleted];
+}
+
void* Context::mark() {
std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
if (impl_->cb_def != nil) split_runs(); // (the streams' pending work goes first, inside)
@@ -388,7 +495,7 @@ void* Context::mark() {
void Context::wait_mark(void* token) {
id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;
const auto tw0 = std::chrono::steady_clock::now();
- [cb waitUntilCompleted];
+ wait_spin(cb);
g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();
++g_wait_n;
if (cb.status != MTLCommandBufferStatusCompleted)
@@ -405,6 +512,7 @@ void Context::flush_runs() {
[impl_->enc_def endEncoding];
if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];
[impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
+ add_busy_handler(impl_->cb_def);
[impl_->cb_def commit];
impl_->inflight = impl_->cb_def;
impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();
@@ -415,7 +523,7 @@ void Context::flush_runs() {
if (impl_->inflight != nil) {
if (std::getenv("DS4_TRACE_WAIT")) { void* fr[8]; const int nf = backtrace(fr, 8); std::fprintf(stderr, "WAIT flush_runs:\n"); backtrace_symbols_fd(fr + 1, nf - 1, 2); }
const auto tw0 = std::chrono::steady_clock::now();
- [impl_->inflight waitUntilCompleted];
+ wait_spin(impl_->inflight);
g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();
++g_wait_n;
{ const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;
@@ -447,6 +555,7 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,
}
const auto te0 = std::chrono::steady_clock::now();
encode(impl_->enc_def, ds[0]);
+ g_last_act_ns.store(ka_now_ns(), std::memory_order_relaxed);
g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();
if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";
g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cu
index 297025f51..ba4ed3e3d 100644
--- a/src/dsv41/ds4_gguf.cu
+++ b/src/dsv41/ds4_gguf.cu
@@ -1093,6 +1093,15 @@ x3::Mat view(const x3::Mat& m, int c0, int nc) {
bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
if (n <= 0 || n > x3::kMaxMvJobs) return false;
+#if defined(STRATA_METAL)
+ // the jobs' int8 activations are quantised in one dispatch before the matvecs (a dispatch a job was 45% of the decode token's)
+ struct Q8Batch { const float* x; uint64_t ldx; uint32_t rows, k; int8_t* q; float* d; } q8_batch[x3::kMaxMvJobs];
+ int nq8 = 0;
+ const bool q8_deferred = std::getenv("DS4_NO_Q8_BATCH") == nullptr && std::getenv("DS4_DBG_Q8") == nullptr;
+#define Q8_QUANT(XF, LDX, ROWS, KK, Q, D, S) do { if (q8_deferred) q8_batch[nq8++] = Q8Batch{XF, (uint64_t) (LDX), (uint32_t) (ROWS), (uint32_t) (KK), Q, D}; else q8_quant(XF, LDX, ROWS, KK, Q, D, S); } while (0)
+#else
+#define Q8_QUANT(XF, LDX, ROWS, KK, Q, D, S) q8_quant(XF, LDX, ROWS, KK, Q, D, S)
+#endif
BfJobs J[3] = {}; // the BF16 jobs, the MXFP4 ones, the Q8_0 ones: a launch each
int nj[3] = {0, 0, 0}, max_n[3] = {0, 0, 0}, max_c[3] = {1, 1, 1};
bool vec[3] = {true, true, true};
@@ -1113,7 +1122,7 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
return false;
}
const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;
- q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);
+ Q8_QUANT(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);
iq::MvJob& b = IJ[nij++];
b.w = (const uint8_t*) m.raw;
b.ld = row_stride(m);
@@ -1165,7 +1174,7 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
#if defined(STRATA_METAL)
if (std::getenv("DS4_DBG_Q8_PRE")) ds4_metal_flush(); // everything before the quantiser has finished
#endif
- q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);
+ Q8_QUANT(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);
#if defined(STRATA_METAL)
if (std::getenv("DS4_DBG_Q8")) { // DBG: the quantiser against a host recompute
ds4_metal_flush();
@@ -1197,6 +1206,9 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {
vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));
}
}
+#if defined(STRATA_METAL)
+ if (nq8 > 0) ::strata::dsv41::metal_host::q8_quant_multi(q8_batch, nq8);
+#endif
for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);
#if defined(STRATA_METAL)
if (std::getenv("DS4_DBG_Q8") && nj[2] > 0) { // DBG: the Q8_0 matvec's first job against a host recompute
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index bbd90ef1a..e03c190ba 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -3552,6 +3552,30 @@ struct Model::Impl {
return true;
}
+#if defined(STRATA_METAL)
+ // Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it
+ // like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).
+ bool zc_try(Layer& l, int e) {
+ static const bool on = std::getenv("DS4_ZC") != nullptr; // (measured: no faster than the CPU lane - cold pages come from the SSD either way)
+ if (!on || l.lay.fmt != gg::kFmtMXFP4 || l.nm != 1 || l.vm.empty()) return false;
+ if (l.vm[(size_t) e][0].raw != nullptr) return true;
+ uint64_t a[3];
+ for (int r = 0; r < 3; ++r) {
+ a[r] = ::strata::dsv41::metal_host::zc_map(mx_mat(l, e, r), mx_part_bytes(l, r));
+ if (a[r] == 0) return false;
+ }
+ for (int r = 0; r < 3; ++r) {
+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];
+ m = x3::Mat{};
+ m.K = gg::expert_type(l.lay, r);
+ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;
+ m.raw = (const void*) a[r];
+ m.k = r < 2 ? g.dim : g.ff;
+ m.n = r < 2 ? g.ff : g.dim;
+ }
+ return true;
+ }
+#endif
void set_vm(Layer& l, int e, int slot) {
uint8_t* b = l.vram_base + (size_t) slot * l.blob;
if (l.lay.fmt == gg::kFmtMXFP4) {
@@ -8256,7 +8280,12 @@ struct Model::Impl {
for (int e = 0; e < l.n_exp; ++e) {
const auto& pe = per[(size_t) e];
if (pe.empty()) continue;
- if (l.slot[(size_t) e] >= 0) {
+#if defined(STRATA_METAL)
+ const bool on_gpu = l.slot[(size_t) e] >= 0 || (T == 1 && zc_try(l, e));
+#else
+ const bool on_gpu = l.slot[(size_t) e] >= 0;
+#endif
+ if (on_gpu) {
vex[nvex] = e;
vrow0[nvex] = nrows;
vn[nvex] = (int) pe.size();
@@ -8316,7 +8345,8 @@ struct Model::Impl {
#if defined(STRATA_METAL)
ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)
#endif
- cpu_experts(l, h_x, ents);
+ static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)
+ if (!skip_cpu) cpu_experts(l, h_x, ents);
std::fill(h_part, h_part + (size_t) T * D, 0.0f);
for (int sl = 0; sl < nslots; ++sl) {
float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;