Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: both queues drain at every sync, engram dequant wraps the strided extent, the ggml CPU lane is in strata_ds4
Niko12210c34a7de3c2d
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txtindex d8b640a7e..c84bde95e 100644--- a/metal/CMakeLists.txt+++ b/metal/CMakeLists.txt@@ -134,12 +134,15 @@ set(STRATA_DS4_SOURCES${DS4}/ds4_main.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/engine/ds4_finish.cpp ${DS4}/ds4_model.cu ${DS4}/ds4_gguf.cu ${DS4}/ds4_iq.cu ${DS4}/ds4_kernels.cu ${DS4}/ds4_tq.cu${DS4}/ds4_vision.cu${DS4}/vendor/exl3.cu ${DS4}/vendor/exl3_cpu.cpp ${DS4}/vendor/moe_win.cu ${DS4}/vendor/sampler.cu- ${DS4}/ds4_gg_cpu.cpp ${DS4}/ds4_gguf_cpu.cpp ${DS4}/ds4_iq_cpu.cpp ${DS4}/ds4_tq_cpu.cpp ${DS4}/ds4_packinfo.cpp)+ ${STRATA_ROOT}/src/kernels/cpu/iq256_portable.cpp ${DS4}/ds4_gg_cpu.cpp ${DS4}/ds4_gguf_cpu.cpp ${DS4}/ds4_iq_cpu.cpp ${DS4}/ds4_tq_cpu.cpp ${DS4}/ds4_packinfo.cpp)set_source_files_properties(${DS4}/ds4_model.cu ${DS4}/ds4_gguf.cu ${DS4}/ds4_iq.cu ${DS4}/ds4_kernels.cu ${DS4}/ds4_tq.cu ${DS4}/ds4_vision.cu${DS4}/vendor/exl3.cu ${DS4}/vendor/moe_win.cu ${DS4}/vendor/sampler.cuPROPERTIES LANGUAGE CXX)list(APPEND STRATA_DS4_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/engine/exl3_legacy_stub.cpp) # v1.1.8 EXL3: absent on Metal+# The CPU lane for the experts the GPU does not hold (ggml-cpu's NEON row dots): without it every non-resident expert contributes+# zero and the residual stream stays the embedding (measured: layers 0-3 changed nothing, then random tokens).+set_source_files_properties(${DS4}/ds4_gg_cpu.cpp PROPERTIES COMPILE_DEFINITIONS "STRATA_DSV41_TQ_CPU=1")add_executable(strata_ds4 ${STRATA_DS4_SOURCES})target_include_directories(strata_ds4 PRIVATE${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat ${STRATA_ROOT}/include ${DS4} ${CMAKE_CURRENT_SOURCE_DIR}/srcdiff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex a39ae9122..2e3da062f 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -19,6 +19,7 @@// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host readextern "C" void ds4_metal_flush();+extern "C" void ds4_gpu_sync_all_streams();extern "C" unsigned long long ds4_dispatch_seq();static unsigned long long g_q8_deduped = 0;@@ -1200,8 +1201,9 @@ bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, freturn g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);};auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };- const size_t per = (size_t) (hd / 32) * 18; // fmt 2: IQ4_NL blocks, 18 bytes a 32 values- mtl::Buffer br = wrap(hr, (size_t) n * per);+ // (the kernel strides rows by hd bytes, as the CUDA kernel does, though an IQ4_NL row only fills hd/32*18 of them:+ // wrapping n * 18/32 * hd left the last rows outside the buffer - garbage scales, measured as inf at row 257 of 288)+ mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);if (!br.handle || !bo.handle) return false;mtl::Pipeline pl;@@ -1985,7 +1987,7 @@ extern "C" void ds4_metal_flush() {// other's work unordered: a kernel could read a feed whose blit had not run yet (emb_rows is a zeroed buffer, so// embed wrote exactly 0.0 into h). The stream's queue goes first - it only carries the feeds at a token boundary,// the readbacks are encoded after this returns.- strata::gpu::stream_synchronize(nullptr);+ ds4_gpu_sync_all_streams();if (g_cx) g_cx->flush_runs();}diff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cppindex 1db6c7a4c..cc9129cb3 100644--- a/metal/src/engine/ds4_host.cpp+++ b/metal/src/engine/ds4_host.cpp@@ -214,6 +214,19 @@ bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc) {d.grid[2] = 1;d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;strata::dsv41::metal_host::shared_cx().run({d});+ strata::dsv41::metal_host::shared_cx().flush_runs();+ if (std::getenv("DS4_DBG_CMP")) {+ const float* hv = (const float*) strata::gpu::host_view(h);+ {+ const size_t nv = (size_t) T * hc * dim;+ size_t nf = 0, nz = 0;+ for (size_t i = 0; i < nv; ++i) { const float v = hv[i]; if (!(v == v)) ++nf; else if (v == 0.0f) ++nz; }+ std::fprintf(stderr, "DBG cmp: after embed n=%.0f nonfinite=%zu zeros=%zu\n", (double) nv, nf, nz);+ }+ std::fprintf(stderr, "DBG cmp: view %g %g %g | h=%g h+1=%g\n",+ hv ? hv[0] : -1.0f, hv ? hv[1] : -1.0f, hv ? hv[2] : -1.0f,+ hv ? hv[(size_t) dim * hc] : -1.0f, hv ? hv[(size_t) dim * hc + 1] : -1.0f);+ }if (std::getenv("DS4_DBG_ARGS")) {mtl::Buffer bp = strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) strata::gpu::host_view(h)) & ~(uintptr_t) 16383), 16384);diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex a4510c0c8..e2ff56907 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -30,6 +30,7 @@#include <map>#include <mutex> // recursive_mutex#include <thread>+#include <algorithm>#include <vector>namespace strata::gpu {@@ -119,8 +120,12 @@ Error fail(Error e) {return e;}+static std::vector<StreamImpl*> g_all_streams; // every stream's queue holds pending copies until it is committed+static std::mutex g_all_streams_mu;+StreamImpl* make_stream() {auto* s = new StreamImpl;+ { std::lock_guard<std::mutex> lk(g_all_streams_mu); g_all_streams.push_back(s); }s->q = rt().dev->newCommandQueue();s->q->addResidencySet(rt().rs);s->tl = rt().dev->newSharedEvent();@@ -294,7 +299,8 @@ void alloc_log(const char* what, size_t n) {Error new_alloc(void** p, size_t bytes, bool host) {Runtime& r = rt();- const size_t n = bytes ? bytes : 1;+ const size_t pg = 16384, want = bytes ? bytes : 1;+ const size_t n = std::getenv("DS4_NO_PAD") ? want : ((want + pg - 1) / pg) * pg;MTL::Buffer* b = r.dev->newBuffer(n, MTL::ResourceStorageModeShared | MTL::ResourceHazardTrackingModeUntracked);if (!b) return fail(kErrorMemoryAllocation);Alloc a{b, b->gpuAddress(), (uint8_t*) b->contents(), n, host};@@ -445,7 +451,26 @@ Error host_device_pointer_raw(void** dev, void* host) {*dev = (void*) (uintptr_t) (a->gpu + off);return kSuccess;}+// A synchronising call is a barrier for BOTH queues: the dispatches deferred in mtl_rt's command buffer have to run before+// the host reads or overwrites what they touch (a window's last layers still pending while the next window's embedding rows+// are copied over the buffer its embed reads, measured: window 2's residual stream was window 1's stale tail).+// commit every stream's pending copies/fills, then wait for all of them (commit first: a stream may wait on another's event)+extern "C" void ds4_gpu_sync_all_streams() {+ std::vector<StreamImpl*> v;+ { std::lock_guard<std::mutex> lk(g_all_streams_mu); v = g_all_streams; }+ for (StreamImpl* s : v) if (!s->rec) flush(s);+ for (StreamImpl* s : v) if (!s->rec) wait_value(s->tl, s->submitted);+}+static thread_local bool t_in_barrier = false;+static void both_queues_barrier() {+ if (t_in_barrier) return;+ t_in_barrier = true;+ ds4_metal_flush();+ t_in_barrier = false;+}+Error memset_device(void* p, int value, size_t bytes) {+ both_queues_barrier();StreamImpl* s = S(nullptr);const Error e = encode_fill(s, p, value, bytes);flush(s);@@ -453,6 +478,7 @@ Error memset_device(void* p, int value, size_t bytes) {return e;}Error memset_async(void* p, int value, size_t bytes, Stream st) {+ if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();StreamImpl* s = S(st);if (s->rec) {Op o;@@ -464,6 +490,7 @@ Error memset_async(void* p, int value, size_t bytes, Stream st) {}Error memcpy_sync(void* dst, const void* src, size_t bytes, CopyKind) {+ both_queues_barrier();StreamImpl* s = S(nullptr);const Error e = encode_copy(s, dst, src, bytes);flush(s);@@ -482,6 +509,7 @@ Error memcpy_peer_async(void* dst, int dst_dev, const void* src, int src_dev, sireturn memcpy_async(dst, src, bytes, kDeviceToDevice, st);}Error memcpy_async(void* dst, const void* src, size_t bytes, CopyKind, Stream st) {+ if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();StreamImpl* s = S(st);if (s->rec) {Op o;@@ -497,12 +525,15 @@ Error stream_destroy(Stream s) {if (!s) return kSuccess;flush(s);wait_value(s->tl, s->submitted);+ { std::lock_guard<std::mutex> lk(g_all_streams_mu);+ g_all_streams.erase(std::remove(g_all_streams.begin(), g_all_streams.end(), s), g_all_streams.end()); }s->tl->release();s->q->release();delete s;return kSuccess;}Error stream_synchronize(Stream st) {+ both_queues_barrier();StreamImpl* s = S(st);flush(s);return wait_value(s->tl, s->submitted);diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex fb6cfe11b..0a72f5c06 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -37,6 +37,7 @@ struct Context::Impl {id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate inid<MTLComputeCommandEncoder> enc_def = nil;int def_n = 0;+ std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoderid<MTLDevice> dev;id<MTLCommandQueue> q;std::vector<id<MTLLibrary>> libs;@@ -300,6 +301,7 @@ static void ds4_report_counts() {extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }void Context::flush_runs() {+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }if (impl_->cb_def == nil) return;++g_cb_count;@@ -315,6 +317,7 @@ void Context::flush_runs() {}double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);@autoreleasepool {if (ds.size() == 1 && reps == 1 && !barrier_between) {if (impl_->cb_def == nil || impl_->def_n >= 2000000) {diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 78f8c7f6b..1f306544a 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4745,6 +4745,18 @@ struct Model::Impl {std::memcpy(d.second.first, r.dst, (size_t) hd);std::memcpy(d.second.second, r.scales, (size_t) hd / 32);}+ if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite+ for (size_t i = 0; i < b.reads.size(); ++i) {+ const EgRead& r = b.reads[i];+ for (int k = 0; k < hd / 32; ++k) {+ uint16_t hb;+ std::memcpy(&hb, r.dst + (size_t) k * 18, 2);+ if ((hb & 0x7c00) == 0x7c00)+ std::fprintf(stderr, "DBG eg bad scale: read %zu/%zu file=%d off=%llu n=%u block=%d half=%04x ok=%d\n", i, b.reads.size(),+ r.file, (unsigned long long) r.off, r.n, k, hb, (int) b.ok[i]);+ }+ }+ }{std::lock_guard<std::mutex> lk(eg_cache_mu);for (size_t i = 0; i < b.reads.size(); ++i) {@@ -7476,9 +7488,25 @@ struct Model::Impl {}// one thread: the layers device after device, the hand-offs through pinned memory and events+ void dbg_nf(const char* tag, int il, const float* dev, size_t n) { // DBG: non-finite count of a device buffer+ ds4_metal_flush();+ const float* hp = (const float*) strata::gpu::host_view(dev);+ size_t nf = 0, first = 0; float mx = 0.0f;+ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) { if (!nf) first = i; ++nf; } else if (std::fabs(v) > mx) mx = std::fabs(v); }+ std::fprintf(stderr, "DBG nf il=%d %s n=%zu nonfinite=%zu first=%zu max=%.4g\n", il, tag, n, nf, first, mx);+ }+ void dbg_h(const char* tag, Dev& d, int T) { // DBG: max |h| of the window's residual stream, after the queue drains+ ds4_metal_flush();+ const float* hp = (const float*) strata::gpu::host_view(d.h);+ float mx = 0.0f; size_t nf = 0;+ const size_t n = (size_t) T * g.hc * g.dim;+ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) ++nf; else if (std::fabs(v) > mx) mx = std::fabs(v); }+ std::fprintf(stderr, "DBG hmax %s T=%d max=%.4g nonfinite=%zu h0=%g\n", tag, T, mx, nf, hp[0]);+ }bool forward_serial(int64_t p0, int T, std::string& err) {const int D = g.dim, H = g.hc;int cur = L[0].dev;+ if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();{Dev& d = devs[(size_t) cur];cudaSetDevice(d.id);@@ -7492,6 +7520,7 @@ struct Model::Impl {std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);}init_pre(d.pre, T, d.st);+ if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);}std::vector<float> dump;if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);@@ -7518,6 +7547,7 @@ struct Model::Impl {cur = l.dev;}if (!tap(devs[(size_t) l.dev], il, T, err)) return false;+ if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);if (!layer(l, 0, p0, T, wplan, err)) return false;if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layersDev& dd = devs[(size_t) l.dev];@@ -7669,11 +7699,14 @@ struct Model::Impl {const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;const ArGroup& ag = groups[(size_t) L0.grp].ar;const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;- if (std::getenv("DS4_DBG_H") && mi == 0 && l.il < 2) { // DBG+ if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBGstd::vector<float> hv((size_t) T * H * D);cudaSetDevice(d.id);ds4_metal_flush();- cudaMemcpy(hv.data(), d.h, hv.size() * 4, cudaMemcpyDeviceToHost);+ {+ const float* hvp = (const float*) strata::gpu::host_view(d.h);+ if (hvp) std::memcpy(hv.data(), hvp, hv.size() * 4);+ }int nf = 0, nzv = 0;float mx = 0.0f;size_t firstnan = 0, lastnan = 0;@@ -7702,8 +7735,24 @@ struct Model::Impl {CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,(size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);+ if (std::getenv("DS4_DBG_EG")) {+ dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);+ const float* xv = (const float*) strata::gpu::host_view(d.eg_x);+ for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)+ if (!std::isfinite(xv[i])) {+ const size_t t = i / ((size_t) cols * ehd), c = (i / ehd) % cols;+ const uint8_t* dv = (const uint8_t*) strata::gpu::host_view(d.eg_rows) + (t * cols + c) * ehd;+ const uint8_t* hv = src + t * per + c * ehd;+ std::fprintf(stderr, "DBG eg bad x: t=%zu c=%zu e=%zu | dev/host row bytes:", t, c, i % ehd);+ for (int b = 0; b < 80; ++b) std::fprintf(stderr, "%s%02x/%02x", b % 18 == 0 ? " |" : " ", dv[b], hv[b]);+ std::fprintf(stderr, "\n");+ break;+ }+ }if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;+ if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);+ if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);}// attentionhc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);@@ -8056,7 +8105,7 @@ struct Model::Impl {for (int i = 0; i < l.k_exp; ++i) {const int e = h_idx[t * kPicks + i];if (e < 0 || e >= l.n_exp) {- err = "router picked expert " + std::to_string(e);+ err = "router picked expert " + std::to_string(e) + " (layer " + std::to_string(l.il) + ", token " + std::to_string(t) + ", pick " + std::to_string(i) + ", of " + std::to_string(l.k_exp) + ")";return false;}++pr[e];