Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: both queues drain at every sync, engram dequant wraps the strided extent, the ggml CPU lane is in strata_ds4

Niko12210c34a7de3c2d
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txt
index d8b640a7e..c84bde95e 100644
--- a/metal/CMakeLists.txt
+++ b/metal/CMakeLists.txt
@@ -134,12 +134,15 @@ set(STRATA_DS4_SOURCES
${DS4}/ds4_main.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/engine/ds4_finish.cpp ${DS4}/ds4_model.cu ${DS4}/ds4_gguf.cu ${DS4}/ds4_iq.cu ${DS4}/ds4_kernels.cu ${DS4}/ds4_tq.cu
${DS4}/ds4_vision.cu
${DS4}/vendor/exl3.cu ${DS4}/vendor/exl3_cpu.cpp ${DS4}/vendor/moe_win.cu ${DS4}/vendor/sampler.cu
- ${DS4}/ds4_gg_cpu.cpp ${DS4}/ds4_gguf_cpu.cpp ${DS4}/ds4_iq_cpu.cpp ${DS4}/ds4_tq_cpu.cpp ${DS4}/ds4_packinfo.cpp)
+ ${STRATA_ROOT}/src/kernels/cpu/iq256_portable.cpp ${DS4}/ds4_gg_cpu.cpp ${DS4}/ds4_gguf_cpu.cpp ${DS4}/ds4_iq_cpu.cpp ${DS4}/ds4_tq_cpu.cpp ${DS4}/ds4_packinfo.cpp)
set_source_files_properties(
${DS4}/ds4_model.cu ${DS4}/ds4_gguf.cu ${DS4}/ds4_iq.cu ${DS4}/ds4_kernels.cu ${DS4}/ds4_tq.cu ${DS4}/ds4_vision.cu
${DS4}/vendor/exl3.cu ${DS4}/vendor/moe_win.cu ${DS4}/vendor/sampler.cu
PROPERTIES LANGUAGE CXX)
list(APPEND STRATA_DS4_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/engine/exl3_legacy_stub.cpp) # v1.1.8 EXL3: absent on Metal
+# The CPU lane for the experts the GPU does not hold (ggml-cpu's NEON row dots): without it every non-resident expert contributes
+# zero and the residual stream stays the embedding (measured: layers 0-3 changed nothing, then random tokens).
+set_source_files_properties(${DS4}/ds4_gg_cpu.cpp PROPERTIES COMPILE_DEFINITIONS "STRATA_DSV41_TQ_CPU=1")
add_executable(strata_ds4 ${STRATA_DS4_SOURCES})
target_include_directories(strata_ds4 PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat ${STRATA_ROOT}/include ${DS4} ${CMAKE_CURRENT_SOURCE_DIR}/src
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index a39ae9122..2e3da062f 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -19,6 +19,7 @@
// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host read
extern "C" void ds4_metal_flush();
+extern "C" void ds4_gpu_sync_all_streams();
extern "C" unsigned long long ds4_dispatch_seq();
static unsigned long long g_q8_deduped = 0;
@@ -1200,8 +1201,9 @@ bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, f
return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
};
auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
- const size_t per = (size_t) (hd / 32) * 18; // fmt 2: IQ4_NL blocks, 18 bytes a 32 values
- mtl::Buffer br = wrap(hr, (size_t) n * per);
+ // (the kernel strides rows by hd bytes, as the CUDA kernel does, though an IQ4_NL row only fills hd/32*18 of them:
+ // wrapping n * 18/32 * hd left the last rows outside the buffer - garbage scales, measured as inf at row 257 of 288)
+ mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);
mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);
if (!br.handle || !bo.handle) return false;
mtl::Pipeline pl;
@@ -1985,7 +1987,7 @@ extern "C" void ds4_metal_flush() {
// other's work unordered: a kernel could read a feed whose blit had not run yet (emb_rows is a zeroed buffer, so
// embed wrote exactly 0.0 into h). The stream's queue goes first - it only carries the feeds at a token boundary,
// the readbacks are encoded after this returns.
- strata::gpu::stream_synchronize(nullptr);
+ ds4_gpu_sync_all_streams();
if (g_cx) g_cx->flush_runs();
}
diff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cpp
index 1db6c7a4c..cc9129cb3 100644
--- a/metal/src/engine/ds4_host.cpp
+++ b/metal/src/engine/ds4_host.cpp
@@ -214,6 +214,19 @@ bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc) {
d.grid[2] = 1;
d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
strata::dsv41::metal_host::shared_cx().run({d});
+ strata::dsv41::metal_host::shared_cx().flush_runs();
+ if (std::getenv("DS4_DBG_CMP")) {
+ const float* hv = (const float*) strata::gpu::host_view(h);
+ {
+ const size_t nv = (size_t) T * hc * dim;
+ size_t nf = 0, nz = 0;
+ for (size_t i = 0; i < nv; ++i) { const float v = hv[i]; if (!(v == v)) ++nf; else if (v == 0.0f) ++nz; }
+ std::fprintf(stderr, "DBG cmp: after embed n=%.0f nonfinite=%zu zeros=%zu\n", (double) nv, nf, nz);
+ }
+ std::fprintf(stderr, "DBG cmp: view %g %g %g | h=%g h+1=%g\n",
+ hv ? hv[0] : -1.0f, hv ? hv[1] : -1.0f, hv ? hv[2] : -1.0f,
+ hv ? hv[(size_t) dim * hc] : -1.0f, hv ? hv[(size_t) dim * hc + 1] : -1.0f);
+ }
if (std::getenv("DS4_DBG_ARGS")) {
mtl::Buffer bp = strata::dsv41::metal_host::shared_cx().wrap(
(void*) (((uintptr_t) strata::gpu::host_view(h)) & ~(uintptr_t) 16383), 16384);
diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cpp
index a4510c0c8..e2ff56907 100644
--- a/metal/src/gpu/runtime.cpp
+++ b/metal/src/gpu/runtime.cpp
@@ -30,6 +30,7 @@
#include <map>
#include <mutex> // recursive_mutex
#include <thread>
+#include <algorithm>
#include <vector>
namespace strata::gpu {
@@ -119,8 +120,12 @@ Error fail(Error e) {
return e;
}
+static std::vector<StreamImpl*> g_all_streams; // every stream's queue holds pending copies until it is committed
+static std::mutex g_all_streams_mu;
+
StreamImpl* make_stream() {
auto* s = new StreamImpl;
+ { std::lock_guard<std::mutex> lk(g_all_streams_mu); g_all_streams.push_back(s); }
s->q = rt().dev->newCommandQueue();
s->q->addResidencySet(rt().rs);
s->tl = rt().dev->newSharedEvent();
@@ -294,7 +299,8 @@ void alloc_log(const char* what, size_t n) {
Error new_alloc(void** p, size_t bytes, bool host) {
Runtime& r = rt();
- const size_t n = bytes ? bytes : 1;
+ const size_t pg = 16384, want = bytes ? bytes : 1;
+ const size_t n = std::getenv("DS4_NO_PAD") ? want : ((want + pg - 1) / pg) * pg;
MTL::Buffer* b = r.dev->newBuffer(n, MTL::ResourceStorageModeShared | MTL::ResourceHazardTrackingModeUntracked);
if (!b) return fail(kErrorMemoryAllocation);
Alloc a{b, b->gpuAddress(), (uint8_t*) b->contents(), n, host};
@@ -445,7 +451,26 @@ Error host_device_pointer_raw(void** dev, void* host) {
*dev = (void*) (uintptr_t) (a->gpu + off);
return kSuccess;
}
+// A synchronising call is a barrier for BOTH queues: the dispatches deferred in mtl_rt's command buffer have to run before
+// the host reads or overwrites what they touch (a window's last layers still pending while the next window's embedding rows
+// are copied over the buffer its embed reads, measured: window 2's residual stream was window 1's stale tail).
+// commit every stream's pending copies/fills, then wait for all of them (commit first: a stream may wait on another's event)
+extern "C" void ds4_gpu_sync_all_streams() {
+ std::vector<StreamImpl*> v;
+ { std::lock_guard<std::mutex> lk(g_all_streams_mu); v = g_all_streams; }
+ for (StreamImpl* s : v) if (!s->rec) flush(s);
+ for (StreamImpl* s : v) if (!s->rec) wait_value(s->tl, s->submitted);
+}
+static thread_local bool t_in_barrier = false;
+static void both_queues_barrier() {
+ if (t_in_barrier) return;
+ t_in_barrier = true;
+ ds4_metal_flush();
+ t_in_barrier = false;
+}
+
Error memset_device(void* p, int value, size_t bytes) {
+ both_queues_barrier();
StreamImpl* s = S(nullptr);
const Error e = encode_fill(s, p, value, bytes);
flush(s);
@@ -453,6 +478,7 @@ Error memset_device(void* p, int value, size_t bytes) {
return e;
}
Error memset_async(void* p, int value, size_t bytes, Stream st) {
+ if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();
StreamImpl* s = S(st);
if (s->rec) {
Op o;
@@ -464,6 +490,7 @@ Error memset_async(void* p, int value, size_t bytes, Stream st) {
}
Error memcpy_sync(void* dst, const void* src, size_t bytes, CopyKind) {
+ both_queues_barrier();
StreamImpl* s = S(nullptr);
const Error e = encode_copy(s, dst, src, bytes);
flush(s);
@@ -482,6 +509,7 @@ Error memcpy_peer_async(void* dst, int dst_dev, const void* src, int src_dev, si
return memcpy_async(dst, src, bytes, kDeviceToDevice, st);
}
Error memcpy_async(void* dst, const void* src, size_t bytes, CopyKind, Stream st) {
+ if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();
StreamImpl* s = S(st);
if (s->rec) {
Op o;
@@ -497,12 +525,15 @@ Error stream_destroy(Stream s) {
if (!s) return kSuccess;
flush(s);
wait_value(s->tl, s->submitted);
+ { std::lock_guard<std::mutex> lk(g_all_streams_mu);
+ g_all_streams.erase(std::remove(g_all_streams.begin(), g_all_streams.end(), s), g_all_streams.end()); }
s->tl->release();
s->q->release();
delete s;
return kSuccess;
}
Error stream_synchronize(Stream st) {
+ both_queues_barrier();
StreamImpl* s = S(st);
flush(s);
return wait_value(s->tl, s->submitted);
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index fb6cfe11b..0a72f5c06 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -37,6 +37,7 @@ struct Context::Impl {
id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate in
id<MTLComputeCommandEncoder> enc_def = nil;
int def_n = 0;
+ std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoder
id<MTLDevice> dev;
id<MTLCommandQueue> q;
std::vector<id<MTLLibrary>> libs;
@@ -300,6 +301,7 @@ static void ds4_report_counts() {
extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }
void Context::flush_runs() {
+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }
if (impl_->cb_def == nil) return;
++g_cb_count;
@@ -315,6 +317,7 @@ void Context::flush_runs() {
}
double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {
+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
@autoreleasepool {
if (ds.size() == 1 && reps == 1 && !barrier_between) {
if (impl_->cb_def == nil || impl_->def_n >= 2000000) {
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 78f8c7f6b..1f306544a 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -4745,6 +4745,18 @@ struct Model::Impl {
std::memcpy(d.second.first, r.dst, (size_t) hd);
std::memcpy(d.second.second, r.scales, (size_t) hd / 32);
}
+ if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite
+ for (size_t i = 0; i < b.reads.size(); ++i) {
+ const EgRead& r = b.reads[i];
+ for (int k = 0; k < hd / 32; ++k) {
+ uint16_t hb;
+ std::memcpy(&hb, r.dst + (size_t) k * 18, 2);
+ if ((hb & 0x7c00) == 0x7c00)
+ std::fprintf(stderr, "DBG eg bad scale: read %zu/%zu file=%d off=%llu n=%u block=%d half=%04x ok=%d\n", i, b.reads.size(),
+ r.file, (unsigned long long) r.off, r.n, k, hb, (int) b.ok[i]);
+ }
+ }
+ }
{
std::lock_guard<std::mutex> lk(eg_cache_mu);
for (size_t i = 0; i < b.reads.size(); ++i) {
@@ -7476,9 +7488,25 @@ struct Model::Impl {
}
// one thread: the layers device after device, the hand-offs through pinned memory and events
+ void dbg_nf(const char* tag, int il, const float* dev, size_t n) { // DBG: non-finite count of a device buffer
+ ds4_metal_flush();
+ const float* hp = (const float*) strata::gpu::host_view(dev);
+ size_t nf = 0, first = 0; float mx = 0.0f;
+ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) { if (!nf) first = i; ++nf; } else if (std::fabs(v) > mx) mx = std::fabs(v); }
+ std::fprintf(stderr, "DBG nf il=%d %s n=%zu nonfinite=%zu first=%zu max=%.4g\n", il, tag, n, nf, first, mx);
+ }
+ void dbg_h(const char* tag, Dev& d, int T) { // DBG: max |h| of the window's residual stream, after the queue drains
+ ds4_metal_flush();
+ const float* hp = (const float*) strata::gpu::host_view(d.h);
+ float mx = 0.0f; size_t nf = 0;
+ const size_t n = (size_t) T * g.hc * g.dim;
+ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) ++nf; else if (std::fabs(v) > mx) mx = std::fabs(v); }
+ std::fprintf(stderr, "DBG hmax %s T=%d max=%.4g nonfinite=%zu h0=%g\n", tag, T, mx, nf, hp[0]);
+ }
bool forward_serial(int64_t p0, int T, std::string& err) {
const int D = g.dim, H = g.hc;
int cur = L[0].dev;
+ if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();
{
Dev& d = devs[(size_t) cur];
cudaSetDevice(d.id);
@@ -7492,6 +7520,7 @@ struct Model::Impl {
std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);
}
init_pre(d.pre, T, d.st);
+ if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);
}
std::vector<float> dump;
if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);
@@ -7518,6 +7547,7 @@ struct Model::Impl {
cur = l.dev;
}
if (!tap(devs[(size_t) l.dev], il, T, err)) return false;
+ if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);
if (!layer(l, 0, p0, T, wplan, err)) return false;
if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers
Dev& dd = devs[(size_t) l.dev];
@@ -7669,11 +7699,14 @@ struct Model::Impl {
const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
const ArGroup& ag = groups[(size_t) L0.grp].ar;
const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
- if (std::getenv("DS4_DBG_H") && mi == 0 && l.il < 2) { // DBG
+ if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG
std::vector<float> hv((size_t) T * H * D);
cudaSetDevice(d.id);
ds4_metal_flush();
- cudaMemcpy(hv.data(), d.h, hv.size() * 4, cudaMemcpyDeviceToHost);
+ {
+ const float* hvp = (const float*) strata::gpu::host_view(d.h);
+ if (hvp) std::memcpy(hv.data(), hvp, hv.size() * 4);
+ }
int nf = 0, nzv = 0;
float mx = 0.0f;
size_t firstnan = 0, lastnan = 0;
@@ -7702,8 +7735,24 @@ struct Model::Impl {
CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
(size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
+ if (std::getenv("DS4_DBG_EG")) {
+ dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);
+ const float* xv = (const float*) strata::gpu::host_view(d.eg_x);
+ for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)
+ if (!std::isfinite(xv[i])) {
+ const size_t t = i / ((size_t) cols * ehd), c = (i / ehd) % cols;
+ const uint8_t* dv = (const uint8_t*) strata::gpu::host_view(d.eg_rows) + (t * cols + c) * ehd;
+ const uint8_t* hv = src + t * per + c * ehd;
+ std::fprintf(stderr, "DBG eg bad x: t=%zu c=%zu e=%zu | dev/host row bytes:", t, c, i % ehd);
+ for (int b = 0; b < 80; ++b) std::fprintf(stderr, "%s%02x/%02x", b % 18 == 0 ? " |" : " ", dv[b], hv[b]);
+ std::fprintf(stderr, "\n");
+ break;
+ }
+ }
if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
+ if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);
engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
+ if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);
}
// attention
hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
@@ -8056,7 +8105,7 @@ struct Model::Impl {
for (int i = 0; i < l.k_exp; ++i) {
const int e = h_idx[t * kPicks + i];
if (e < 0 || e >= l.n_exp) {
- err = "router picked expert " + std::to_string(e);
+ err = "router picked expert " + std::to_string(e) + " (layer " + std::to_string(l.il) + ", token " + std::to_string(t) + ", pick " + std::to_string(i) + ", of " + std::to_string(l.k_exp) + ")";
return false;
}
++pr[e];