Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: the ring backup flag, decode takes the host-routed path, split commits sync the streams first, sparse_attn sizes its binding without a host read of device data
Niko1221860e384db78e
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 6678943ae..8ad2eea5f 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -19,6 +19,7 @@// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host readextern "C" void ds4_metal_flush();+extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu);extern "C" void ds4_gpu_sync_all_streams();extern "C" unsigned long long ds4_dispatch_seq();static unsigned long long g_q8_deduped = 0;@@ -956,7 +957,7 @@ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d,dsp.p = &pl;dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bb, b_off),mtl::Arg::V((long) p0), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) T),- mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};+ mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes), mtl::Arg::V((unsigned) (hb ? 1 : 0))};dsp.grid[0] = (uint32_t) T;dsp.grid[1] = dsp.grid[2] = 1;dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;@@ -1021,14 +1022,10 @@ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const intif (ar.gen != 0 && ar.R > 0) {crows = (size_t) ((ar.p0 + ar.t0 + T + 1) / ar.R) + 2; // every selected row is below this position} else {- int maxrow = -1; // the selected rows, not the cache's whole cap- if (hs != nullptr && sel_stride > 0)- for (int t = 0; t < T; ++t)- for (int i = 0; i < ar.nsel[t]; ++i) {- const int r = ((const int*) hs)[(size_t) t * (size_t) sel_stride + (size_t) i];- if (r > maxrow) maxrow = r;- }- crows = (size_t) (maxrow + 1);+ // (the rows the selection names are on the device and not read back here: an earlier build read them on the+ // host while the kernel that wrote them was still queued, and sized the binding from stale zeros)+ const size_t rem = ds4_gpu_alloc_remaining(hc);+ crows = rem > 0 ? rem / 288 : (size_t) (ar.nsel[0] > 0 ? ar.nsel[0] : 1);}if (crows < 1) crows = 1;} else {diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex 2bdae3bf9..665087cb7 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -611,11 +611,11 @@ static inline uint ds4_e4m3_code(float v, int k) {kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device uchar* ring [[buffer(1)]],device uchar* backup [[buffer(2)]], constant long& p0 [[buffer(3)]],constant uint& n [[buffer(4)]], constant uint& T [[buffer(5)]],- constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]],+ constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]], constant uint& has_backup [[buffer(8)]],uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {if (T - t > n) return; // overwritten by a later row of the window anywaydevice uchar* row = ring + (size_t) ((size_t(p0) + t) % n) * rb;- if (backup != nullptr) {+ if (has_backup != 0u) { // (a bound buffer is never a null pointer here: the host says whether there is a backup)for (size_t i = tid; i < rb; i += 256u) backup[(size_t) t * rb + i] = row[i];threadgroup_barrier(mem_flags::mem_threadgroup);}diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex 0735ef955..788dd7654 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -711,3 +711,11 @@ extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* o// dereference raw gpuAddresses (weights, ring buffers, job arrays) that no setBuffer names, and memory that is not resident// reads as zeros.extern "C" void* ds4_gpu_residency_set() { return (void*) strata::gpu::residency_set_impl(); }++// bytes from `cpu` to the end of the allocation that holds it (0: not an allocation)+extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu) {+ size_t off = 0;+ void* b = strata::gpu::buffer_for_cpu_impl(cpu, 1, &off);+ if (!b) return 0;+ return ((MTL::Buffer*) b)->length() - off;+}diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 3bfafe53f..440d4fcd0 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -335,6 +335,9 @@ extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));void Context::split_runs() {+ // (the streams' pending copies come first, as in a full flush: they are queued on another MTLCommandQueue and would+ // otherwise not have run when the commands committed here read what they write)+ if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();// Commit the batched dispatches as their own command buffer WITHOUT waiting: the queue runs buffers in order and makes each// one's writes visible to the next. (Measured: a producer kernel's output was read stale by the next dispatch of the same// buffer for two kernels in a layer; ending the buffer after them is enough, no wait needed.)diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 0c51ea881..bfdca034e 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -920,7 +920,13 @@ struct Model::Impl {std::atomic<int> layer_of[kRing];std::thread service;std::atomic<bool> quit{false};+#if defined(STRATA_METAL)+ // (the async decode has the GPU wait on flags the CPU lane sets while it runs: the Metal queue cannot be waited on in the+ // middle of a command buffer, so a decode step takes the host-routed path, as the prompt windows do)+ bool sync_decode = std::getenv("DS4_ASYNC") == nullptr;+#elsebool sync_decode = std::getenv("DS4_SYNC") != nullptr;+#endif// a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run// before the wait for the CPU lane - the same bits, no faster)bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;@@ -1175,8 +1181,12 @@ struct Model::Impl {if (x == il) return true;return false;}+ bool dump_s_name(const char* name) const { // DS4_DUMP_S_NAMES=a,b,c: only these points (default all)+ static const std::string names = [] { const char* e = std::getenv("DS4_DUMP_S_NAMES"); return std::string(e ? "," : "") + (e ? e : "") + (e ? "," : ""); }();+ return names.empty() || names.find(std::string(",") + name + ",") != std::string::npos;+ }void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {- if (!dump_s_on(il)) return;+ if (!dump_s_on(il) || !dump_s_name(name)) return;std::vector<uint8_t> v(nbytes);cudaSetDevice(d.id);#if defined(STRATA_METAL)@@ -1187,7 +1197,7 @@ struct Model::Impl {dump_sub_host(name, il, p0, T, v.data(), nbytes);}void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {- if (!dump_s_on(il)) return;+ if (!dump_s_on(il) || !dump_s_name(name)) return;std::lock_guard<std::mutex> lk(dump_mu);char nm[16] = {};std::strncpy(nm, name, 15);@@ -7744,6 +7754,7 @@ struct Model::Impl {const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;const ArGroup& ag = groups[(size_t) L0.grp].ar;const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;+ if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBGstd::vector<float> hv((size_t) T * H * D);cudaSetDevice(d.id);@@ -7806,6 +7817,7 @@ struct Model::Impl {if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);}dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);+ dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));// attentionhc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);@@ -7897,6 +7909,8 @@ struct Model::Impl {CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));}compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);+ dump_sub(d, "ckr", l.il, p0, T, d.ckr, (size_t) T * HD * 4);+ dump_sub(d, "cgr", l.il, p0, T, d.cgr, (size_t) T * HD * 4);}}Rows lim;@@ -8001,6 +8015,8 @@ struct Model::Impl {xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);}}+ if (ng > 0) dump_sub(d, "latent", l.il, p0, ng, d.latent, (size_t) ng * HD * 4);+ dump_sub(d, "topk", l.il, p0, T, d.topk, (size_t) T * topk_stride() * 4);for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;ckv = sr.ckv[(size_t) l.dev];}