Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: finish dst stride, BF16 ring pack, residency on the context queue, split the command buffer after fp8 finishes; ds4_launch.h makes the CUDA build work again; more DS4_DUMP_S points
Niko1221193529239cec
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 48225b4ec..561c64ba3 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -775,12 +775,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)endif()target_include_directories(strata_dsv41_kernels PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})+ target_compile_options(strata_dsv41_kernels PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-include ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41/ds4_launch.h>)add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cusrc/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cppsrc/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_packinfo.cpp)target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})+ target_compile_options(strata-dsv41 PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-include ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41/ds4_launch.h>)find_package(Threads REQUIRED)target_link_libraries(strata-dsv41 PRIVATE Threads::Threads)if(WIN32 AND STRATA_ENABLE_HIP) # cudaMemGetInfo is the DXGI-budgeted hip_compat::mem_get_info (src/core/device.cu): on a Strix Halo thediff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 1cc29a3f7..6678943ae 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -159,6 +159,17 @@ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {d.tg[0] = 256;d.tg[1] = d.tg[2] = 1;g_cx->run({d});+ // The fp8-rounding finishes (the KV rows, the attention output) are read stale by the next dispatch when it shares their+ // command buffer: end the buffer after them (committed, not waited for: the queue orders the buffers).+ static const bool no_split = std::getenv("DS4_NO_FINISH_SPLIT") != nullptr;+ if (f.quant && !no_split) g_cx->split_runs();+ if (const char* fm2 = std::getenv("DS4_FLUSH_FINISH_CTX")) { if (f.quant) { if (std::atoi(fm2) == 2) g_cx->split_runs(); else g_cx->flush_runs(); } }+ if (const char* fm = std::getenv("DS4_FLUSH_FINISH")) { // debugging: which finish calls need a drain after them+ const int m = std::atoi(fm);+ if ((m == 1 && f.dst) || (m == 2 && f.rd > 0) || (m == 3 && f.w) || (m == 4 && f.quant) || (m == 5 && !f.dst) ||+ (m == 6 && f.dst == nullptr && f.rd == 0) || (m == 7 && f.rd > 0 && f.inverse) || (m == 8 && f.rd > 0 && !f.inverse && !f.dst) || (m == 9 && f.rd > 0 && !f.dst) || (m == 10 && (f.dst || f.inverse)) || (m == 11 && f.rd > 0 && !f.inverse))+ ds4_metal_flush();+ }return true;}diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex e07af11f7..2bdae3bf9 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -181,7 +181,7 @@ kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[buif (e >= len) continue;if (f.dst != nullptr) {const uint bits = as_type<uint>(r[j]);- f.dst[e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even+ f.dst[(long) b * f.dst_stride + (long) e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even, the vector's own row} else {v[e] = r[j];}@@ -622,7 +622,7 @@ kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device ucharconst device ushort* src = ext + (size_t) (n + t) * d;for (uint e0 = 0; e0 < d; e0 += 256u) {const uint e = e0 + tid;- const float v = e < d ? float(as_type<half>(src[e])) : 0.0f;+ const float v = e < d ? as_type<float>(uint(src[e]) << 16) : 0.0f; // (BF16 bits, not half)uint a = as_type<uint>(v) & 0x7fffffffu; // magnitudes ordered as bitsfor (uint o = 16u; o > 0u; o >>= 1u) a = max(a, simd_shuffle_xor(a, o));const int k = a == 0u ? 0 : int(a >> 23) - 135 + (((a & 0x7fffffu) > 0x600000u) ? 1 : 0);diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex 70772bda8..0735ef955 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -322,6 +322,7 @@ Error free_alloc(void* p) {size_t off = 0;const Alloc* a = find(p, &off);if (!a || off != 0) return fail(kErrorInvalidValue);+ if (std::getenv("DS4_TRACE_FREE")) std::fprintf(stderr, "gpu free %zu bytes (host=%d) at %p\n", a->bytes, (int) a->host, p);device_synchronize(); // as cudaFree: the GPU is done with itstd::lock_guard<std::mutex> lk(r.mu);MTL::Buffer* b = a->buf;@@ -349,6 +350,8 @@ void* buffer_for_cpu_impl(const void* cpu, size_t bytes, size_t* off) {return (void*) a.buf;}+MTL::ResidencySet* residency_set_impl() { return rt().rs; }+void* host_view(const void* p) {size_t off = 0;const Alloc* a = find(p, &off);@@ -703,3 +706,8 @@ Stats stats() { return rt().stats; }extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) {return strata::gpu::buffer_for_cpu_impl(cpu, bytes, off);}++// The residency set every strata::gpu stream queue carries: the Context's queue (mtl_rt) needs it too, because kernels+// dereference raw gpuAddresses (weights, ring buffers, job arrays) that no setBuffer names, and memory that is not resident+// reads as zeros.+extern "C" void* ds4_gpu_residency_set() { return (void*) strata::gpu::residency_set_impl(); }diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.hindex f09185c80..1d47c99ca 100644--- a/metal/src/mtl_rt.h+++ b/metal/src/mtl_rt.h@@ -101,6 +101,7 @@ public:// Encodes `ds` into ONE compute encoder (serial or concurrent dispatch type) and ONE command buffer, `reps`// times over; commits and waits. Returns the GPU time (GPUEndTime - GPUStartTime) in seconds.+ void split_runs(); // commit the batched dispatches as their own command buffer, without waitingvoid flush_runs(); // commit and wait for the dispatches batched since the last flushdouble run(const std::vector<Dispatch>& ds, int reps = 1, bool concurrent = false, bool barrier_between = false);// A dependency graph in ONE concurrent encoder: a memory barrier only before dispatches with `barrier` set, sodiff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 56e3319c6..3bfafe53f 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -33,10 +33,12 @@ std::string read_file(const std::string& path) {return ss.str();}+extern "C" void* ds4_gpu_residency_set() __attribute__((weak_import));struct Context::Impl {id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate inid<MTLComputeCommandEncoder> enc_def = nil;int def_n = 0;+ id<MTLCommandBuffer> inflight = nil; // the last committed-but-not-waited deferred buffer (the queue runs them in order)std::vector<void*> graveyard; // buffers freed while the deferred command buffer still references them (it holds none)std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoderid<MTLDevice> dev;@@ -64,6 +66,10 @@ Context::Context() : impl_(new Impl) {std::exit(2);}impl_->q = [impl_->dev newCommandQueue];+ if (ds4_gpu_residency_set && !std::getenv("DS4_NO_RESIDENCY")) {+ id<MTLResidencySet> rs = (__bridge id<MTLResidencySet>)ds4_gpu_residency_set();+ if (rs) [impl_->q addResidencySet:rs];+ }// env, then <exe dir>/metal/ (a component package), then the compiled-in source-tree pathggml_common_path = strata_metal::resource("STRATA_GGML_COMMON", "ggml-common.h", STRATA_GGML_COMMON_DEFAULT);kernel_dir = strata_metal::resource("STRATA_METAL_KERNELS", "kernels", STRATA_METAL_KERNELS_DEFAULT);@@ -267,7 +273,7 @@ void Context::free(Buffer& b) {// is still queued is read after it is gone (or after another alloc took its memory: the job arrays of iq_mv read the// NEXT call's jobs, measured as rows of zeros and different tokens every run). Hold it until the buffer has completed.std::lock_guard<std::recursive_mutex> lk_(impl_->mu);- const bool hold = impl_->cb_def != nil;+ const bool hold = impl_->cb_def != nil || impl_->inflight != nil;if (b.chunks) {for (void* h : *b.chunks) {if (hold) impl_->graveyard.push_back(h);@@ -294,6 +300,8 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {if (a.buf) {size_t o;id<MTLBuffer> mb = BO(*a.buf, a.off, o);+ static const bool chkbuf = std::getenv("DS4_CHECK_BUF") != nullptr;+ if (chkbuf && o >= [mb length]) std::fprintf(stderr, "mtl: %s arg %zu: offset %zu past the buffer (%zu bytes; Buffer.bytes %zu bias %zu off %zu)\n", d.p->name.c_str(), i, o, (size_t) [mb length], a.buf->bytes, a.buf->bias, (size_t) a.off);[enc setBuffer:mb offset:o atIndex:i];}else@@ -326,21 +334,41 @@ static void ds4_report_counts() {extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));-void Context::flush_runs() {- if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);+void Context::split_runs() {+ // Commit the batched dispatches as their own command buffer WITHOUT waiting: the queue runs buffers in order and makes each+ // one's writes visible to the next. (Measured: a producer kernel's output was read stale by the next dispatch of the same+ // buffer for two kernels in a layer; ending the buffer after them is enough, no wait needed.)std::lock_guard<std::recursive_mutex> lk_(impl_->mu);- if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }if (impl_->cb_def == nil) return;++g_cb_count;- { const double d = impl_->cb_def.GPUEndTime - impl_->cb_def.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0; }[impl_->enc_def endEncoding];[impl_->cb_def commit];- [impl_->cb_def waitUntilCompleted];- if (impl_->cb_def.status != MTLCommandBufferStatusCompleted)- std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->cb_def.error localizedDescription] UTF8String]);+ impl_->inflight = impl_->cb_def;impl_->cb_def = nil;impl_->enc_def = nil;impl_->def_n = 0;+}++void Context::flush_runs() {+ if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);+ if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }+ if (impl_->cb_def != nil) {+ ++g_cb_count;+ [impl_->enc_def endEncoding];+ [impl_->cb_def commit];+ impl_->inflight = impl_->cb_def;+ impl_->cb_def = nil;+ impl_->enc_def = nil;+ impl_->def_n = 0;+ }+ if (impl_->inflight != nil) {+ [impl_->inflight waitUntilCompleted];+ { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0; }+ if (impl_->inflight.status != MTLCommandBufferStatusCompleted)+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);+ impl_->inflight = nil;+ }for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }impl_->graveyard.clear();}@@ -370,7 +398,9 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,++impl_->def_n;++g_disp_count;static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr; // debugging: every dispatch alone, in order- if (each) {+ static const std::string only = std::getenv("DS4_FLUSH_AFTER") ? std::getenv("DS4_FLUSH_AFTER") : ""; // ",name1,name2,"+ const bool listed = !only.empty() && only.find("," + ds[0].p->name + ",") != std::string::npos;+ if (each || listed) {if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();flush_runs();}diff --git a/src/dsv41/ds4_launch.h b/src/dsv41/ds4_launch.hnew file mode 100644index 000000000..7d0bc9ad8--- /dev/null+++ b/src/dsv41/ds4_launch.h@@ -0,0 +1,28 @@+// strata_ds4_launch(kernel, grid, block, shared, stream, args...): the engine's one spelling of a kernel launch. On CUDA it is+// kernel<<<grid, block, shared, stream>>>(args...); on Metal metal/include/metal_compat/cuda_launch.h maps the same call onto+// a dispatch. (Force-included for CUDA compiles by CMakeLists.txt.)+#pragma once+#if defined(__CUDACC__) && !defined(STRATA_METAL)+#include <cuda_runtime.h>+#include <cstdio>+#include <cstdlib>+template <class... KA, class... CA>+inline void strata_ds4_launch_at(const char* file, int line, void (*kernel)(KA...), dim3 grid, dim3 block, size_t shared,+ cudaStream_t stream, CA&&... args) {+ kernel<<<grid, block, shared, stream>>>(static_cast<KA>(args)...);+ static const bool check = std::getenv("DS4_LAUNCH_CHECK") != nullptr; // report the launch that failed (not a later call)+ if (check) {+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess)+ std::fprintf(stderr, "ds4 launch failed at %s:%d: %s (grid %u,%u,%u block %u,%u,%u shared %zu)\n", file, line,+ cudaGetErrorString(e), grid.x, grid.y, grid.z, block.x, block.y, block.z, shared);+ }+}+#define strata_ds4_launch(...) strata_ds4_launch_at(__FILE__, __LINE__, __VA_ARGS__)++// (the Metal debug dumps in ds4_model.cu name these; on CUDA there is no deferred queue and a device pointer is not a host view)+inline void ds4_metal_flush() {}+namespace strata::gpu {+inline void* host_view(const void* p) { return const_cast<void*>(p); }+}+#endifdiff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 240422119..0c51ea881 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -7780,6 +7780,8 @@ struct Model::Impl {CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,(size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);+ dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);+ dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);if (std::getenv("DS4_DBG_EG")) {dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);const float* xv = (const float*) strata::gpu::host_view(d.eg_x);@@ -7796,7 +7798,11 @@ struct Model::Impl {}if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);+ dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);+ dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);+ dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);+ dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);}dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);@@ -7852,6 +7858,7 @@ struct Model::Impl {finish(d.kvr, T, HD, HD, fk, s);if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);+ dump_sub(d, "ext", l.il, p0, T, d.ext, ((size_t) g.win + T) * HD * 2); // (BF16 rows: the ring's, then this window's)AttnRows ar;for (int t = 0; t < T; ++t) {if (l.mtp) {