Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: bind the allocation's own MTLBuffer instead of a no-copy alias per call (the alias read stale data), deferred frees, matvec jobs cut at 8 activations, Q8/quant debug checks
Niko12219f45341966bc
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 2e3da062f..1cc29a3f7 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -485,9 +485,50 @@ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float// ds4_gguf.cu:371 - the MXFP4 expert matvec. The engine builds its BfJobs array on the HOST (with device pointers inside), so// the jobs go to the kernel as an opaque constant copy: same bytes, same layout, indexed by threadgroup_position_in_grid.y.-bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt) {- if (nj <= 0 || max_n <= 0) return true;+namespace {+// mirrors ds4_gguf.cu's BfJob (and the kernels' Ds4MxJob)+struct HostBfJob {+ const void* w;+ uint64_t ld;+ const float* x;+ uint64_t xcs;+ float* y;+ uint64_t ycs;+ const float* bias;+ float alpha;+ int k, n, ncol;+ const int8_t* xq;+ const float* xd;+};+} // namespace++bool mx_mv(const void* jobs_in, int nj_in, int max_n, size_t job_bytes, int fmt) {+ if (nj_in <= 0 || max_n <= 0) return true;if (!ensure()) return false;+ // The matvec kernels keep 8 accumulators (the T=8 window): a job of more activations (a 16-token window) is cut into+ // jobs of at most 8, each reading its own activations and writing its own rows of y. (Left whole, the columns past the+ // eighth were never computed: rows of zeros in every projection of a prompt window.)+ std::vector<HostBfJob> cut;+ if (job_bytes == sizeof(HostBfJob)) {+ const HostBfJob* src = (const HostBfJob*) jobs_in;+ for (int i = 0; i < nj_in; ++i) {+ const HostBfJob& q = src[i];+ const int ncol = q.ncol > 0 ? q.ncol : 1;+ for (int c0 = 0; c0 < ncol; c0 += 8) {+ HostBfJob b = q;+ b.ncol = std::min(8, ncol - c0);+ if (c0 != 0) {+ if (q.x) b.x = q.x + (size_t) c0 * (size_t) q.xcs;+ b.y = q.y + (size_t) c0 * (size_t) q.ycs;+ if (q.xq) b.xq = q.xq + (size_t) c0 * (size_t) q.k;+ if (q.xd) b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);+ }+ cut.push_back(b);+ }+ }+ }+ const void* jobs = cut.empty() ? jobs_in : (const void*) cut.data();+ const int nj = cut.empty() ? nj_in : (int) cut.size();mtl::Buffer bj = g_cx->alloc((size_t) nj * job_bytes);if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: mx_mv job buffer failed\n"); return false; }std::memcpy(bj.ptr, jobs, (size_t) nj * job_bytes);diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex e2ff56907..70772bda8 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -337,6 +337,18 @@ Error free_alloc(void* p) {} // namespace// ---------------------------------------------------------------------------------------------------- runtime.hpp+void* buffer_for_cpu_impl(const void* cpu, size_t bytes, size_t* off) {+ Runtime& r = rt();+ std::lock_guard<std::mutex> lk(r.mu);+ auto ic = r.by_cpu.upper_bound((uintptr_t) cpu);+ if (ic == r.by_cpu.begin()) return nullptr;+ --ic;+ const Alloc& a = r.by_gpu[ic->second];+ if ((uintptr_t) cpu < ic->first || (uintptr_t) cpu + bytes > ic->first + a.bytes) return nullptr;+ *off = (uintptr_t) cpu - ic->first;+ return (void*) a.buf;+}+void* host_view(const void* p) {size_t off = 0;const Alloc* a = find(p, &off);@@ -687,3 +699,7 @@ Stats stats() { return rt().stats; }} // namespace metal} // namespace strata::gpu++extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) {+ return strata::gpu::buffer_for_cpu_impl(cpu, bytes, off);+}diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.hindex 136dab06e..f09185c80 100644--- a/metal/src/mtl_rt.h+++ b/metal/src/mtl_rt.h@@ -22,6 +22,7 @@ struct Buffer {// a buffer longer than maxBufferLength: overlapping chunks every `stride` bytes (see Context::wrap)std::shared_ptr<std::vector<void*>> chunks;size_t stride = 0;+ size_t bias = 0; // bytes from the MTLBuffer's start to ptr (a view onto an allocation the runtime owns)template <class T> T* as() const { return static_cast<T*>(ptr); }};diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 132195d83..56e3319c6 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -37,6 +37,7 @@ struct Context::Impl {id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate inid<MTLComputeCommandEncoder> enc_def = nil;int def_n = 0;+ std::vector<void*> graveyard; // buffers freed while the deferred command buffer still references them (it holds none)std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoderid<MTLDevice> dev;id<MTLCommandQueue> q;@@ -48,7 +49,7 @@ struct Context::Impl {static inline id<MTLBuffer> B(const Buffer& b) { return (__bridge id<MTLBuffer>)b.handle; }// the MTLBuffer and offset that bind byte `off` of b (a chunked buffer: the chunk the offset selects)static inline id<MTLBuffer> BO(const Buffer& b, size_t off, size_t& o) {- if (!b.chunks) { o = off; return B(b); }+ if (!b.chunks) { o = off + b.bias; return B(b); }const size_t k = std::min(off / b.stride, b.chunks->size() - 1);o = off - k * b.stride;return (__bridge id<MTLBuffer>)(*b.chunks)[k];@@ -231,7 +232,23 @@ Buffer Context::wrap(void* p, size_t bytes, size_t max_span) {return r;}+// The runtime's own MTLBuffer under a CPU address (strata::gpu's allocations), when [p, p + bytes) lies inside one. Binding it+// - the same MTLBuffer, so the same GPU address, as every other kernel that touches this memory - instead of a fresh no-copy+// buffer per call. (Each fresh wrapper is a second GPU mapping of the same pages: the quantiser read the activations through+// one mapping while the norm that produced them had written through another, and saw the previous layer's.)+extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) __attribute__((weak_import));Buffer Context::wrap(void* p, size_t bytes) {+ if (ds4_gpu_buffer_for_cpu && !std::getenv("DS4_WRAP_COPY")) {+ size_t off = 0;+ if (void* mb = ds4_gpu_buffer_for_cpu(p, bytes, &off)) {+ Buffer r;+ r.handle = (__bridge_retained void*)((__bridge id<MTLBuffer>)mb);+ r.ptr = p;+ r.bytes = bytes;+ r.bias = off;+ return r;+ }+ }id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];if (!b) {std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);@@ -246,14 +263,22 @@ Buffer Context::wrap(void* p, size_t bytes) {}void Context::free(Buffer& b) {+ // The deferred command buffer is made WITHOUT retained references: a buffer released here while a dispatch that names it+ // is still queued is read after it is gone (or after another alloc took its memory: the job arrays of iq_mv read the+ // NEXT call's jobs, measured as rows of zeros and different tokens every run). Hold it until the buffer has completed.+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);+ const bool hold = impl_->cb_def != nil;if (b.chunks) {- for (void* h : *b.chunks) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }+ for (void* h : *b.chunks) {+ if (hold) impl_->graveyard.push_back(h);+ else { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }+ }b = Buffer{};return;}if (b.handle) {- id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)b.handle;- x = nil;+ if (hold) impl_->graveyard.push_back(b.handle);+ else { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)b.handle; x = nil; }}b = Buffer{};}@@ -302,6 +327,7 @@ extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));void Context::flush_runs() {+ if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);std::lock_guard<std::recursive_mutex> lk_(impl_->mu);if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }if (impl_->cb_def == nil) return;@@ -315,6 +341,8 @@ void Context::flush_runs() {impl_->cb_def = nil;impl_->enc_def = nil;impl_->def_n = 0;+ for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }+ impl_->graveyard.clear();}double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {@@ -324,9 +352,21 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,if (impl_->cb_def == nil || impl_->def_n >= 2000000) {flush_runs();impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];- impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial];+ impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];}encode(impl_->enc_def, ds[0]);+ static const bool trace = std::getenv("DS4_TRACE_DISP") != nullptr;+ if (trace) { std::lock_guard<std::mutex> lk(g_hist_mu); std::fprintf(stderr, "TRACE disp %s\n", ds[0].p->name.c_str()); }+ // Every wrapper binds its own MTLBuffer over the same memory, so Metal's hazard tracking cannot see that one+ // dispatch reads what the previous wrote: without an explicit barrier the reader saw the stale value (measured: the+ // second matvec's activation quantiser read the previous layer's x).+ static const bool no_mb = std::getenv("DS4_NO_MBARRIER") != nullptr;+ if (!no_mb) [impl_->enc_def memoryBarrierWithScope:MTLBarrierScopeBuffers];+ static const bool enc_each = std::getenv("DS4_ENC_EACH") != nullptr; // a new encoder per dispatch (debugging)+ if (enc_each) {+ [impl_->enc_def endEncoding];+ impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:(std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial)];+ }++impl_->def_n;++g_disp_count;static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr; // debugging: every dispatch alone, in orderdiff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 892f6f9d8..caec54b39 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1161,13 +1161,69 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {const size_t at = (size_t) i * x3::kMaxMvCols * kMaxQK;b.xq = sc->mq + at;b.xd = sc->md + at / 32;+#if defined(STRATA_METAL)+ if (std::getenv("DS4_DBG_Q8_PRE")) ds4_metal_flush(); // everything before the quantiser has finished+#endifq8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);+#if defined(STRATA_METAL)+ if (std::getenv("DS4_DBG_Q8")) { // DBG: the quantiser against a host recompute+ ds4_metal_flush();+ if (std::getenv("DS4_DBG_Q8_TWICE")) { // the same dispatch again: does the first one get lost?+ q8_quant(q.xf, q.ncol > 1 ? q.xf_cs : (size_t) m.k, q.ncol, m.k, sc->mq + at, sc->md + at / 32, s);+ ds4_metal_flush();+ }+ const float* xf = (const float*) strata::gpu::host_view(q.xf);+ const int8_t* xq = (const int8_t*) strata::gpu::host_view(sc->mq + at);+ const float* xd = (const float*) strata::gpu::host_view(sc->md + at / 32);+ const size_t cs = q.ncol > 1 ? q.xf_cs : (size_t) m.k;+ int bad = 0;+ for (int c = 0; c < q.ncol && bad < 3; ++c)+ for (int kb = 0; kb < m.k / 32 && bad < 3; ++kb) {+ float am = 0.0f;+ for (int j = 0; j < 32; ++j) am = std::max(am, std::fabs(xf[c * cs + kb * 32 + j]));+ const float sc_ = am / 127.0f;+ if (std::fabs(xd[(size_t) c * (m.k / 32) + kb] - sc_) > 1e-6f * (sc_ + 1e-30f)) {+ std::fprintf(stderr, "DBG q8quant mismatch job %d col %d blk %d: scale host %g got %g (x0=%g xq0=%d)\n", i, c, kb, sc_,+ xd[(size_t) c * (m.k / 32) + kb], xf[c * cs + kb * 32], (int) xq[(size_t) c * m.k + kb * 32]);+ ++bad;+ }+ }+ std::fprintf(stderr, "DBG q8quant job %d ncol %d k %d xf=%p xcs=%zu mismatches(first 3)=%d\n", i, q.ncol, m.k, (const void*) q.xf, cs, bad);+ }+#endifvec[f] = vec[f] && b.ld % 4 == 0 && ((uintptr_t) b.w & 3u) == 0 && (m.k / 32) % 2 == 0;} else {vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));}}for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);+#if defined(STRATA_METAL)+ if (std::getenv("DS4_DBG_Q8") && nj[2] > 0) { // DBG: the Q8_0 matvec's first job against a host recompute+ ds4_metal_flush();+ const BfJob& b = J[2].j[0];+ const uint8_t* w = (const uint8_t*) strata::gpu::host_view(b.w);+ const int8_t* xq = (const int8_t*) strata::gpu::host_view(b.xq);+ const float* xd = (const float*) strata::gpu::host_view(b.xd);+ const float* y = (const float*) strata::gpu::host_view(b.y);+ const float* xf = (const float*) strata::gpu::host_view(b.x);+ for (int c = 0; c < std::min(b.ncol, 2); ++c)+ for (int r = 0; r < 2; ++r) {+ double acc = 0.0, accf = 0.0;+ for (int kb = 0; kb < b.k / 32; ++kb) {+ const uint8_t* p = w + (size_t) r * b.ld + (size_t) kb * 34;+ uint16_t dh;+ std::memcpy(&dh, p, 2);+ float dw = __half2float(*(const __half*) &dh);+ int sum = 0;+ for (int j = 0; j < 32; ++j) sum += (int) (int8_t) p[2 + j] * (int) xq[(size_t) c * b.k + kb * 32 + j];+ acc += (double) dw * xd[(size_t) c * (b.k / 32) + kb] * sum;+ for (int j = 0; j < 32; ++j) accf += (double) dw * (int) (int8_t) p[2 + j] * xf[(size_t) c * b.xcs + kb * 32 + j];+ }+ std::fprintf(stderr, "DBG q8 job0 k=%d n=%d ncol=%d ld=%zu xcs=%zu c=%d r=%d recompute=%.5g exactf=%.5g kernel_y=%.5g xd0=%g xq0=%d\n", b.k, b.n, b.ncol,+ b.ld, b.xcs, c, r, acc, accf, y[(size_t) c * b.ycs + r], xd[(size_t) c * (b.k / 32)], (int) xq[(size_t) c * b.k]);+ }+ }+#endifif (nij > 0 && !iq::mv(IJ, nij, s)) {fail("mv: a low-bit job not covered");return false;