Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: finish dst stride, BF16 ring pack, residency on the context queue, split the command buffer after fp8 finishes; ds4_launch.h makes the CUDA build work again; more DS4_DUMP_S points

Niko1221193529239cec
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 48225b4ec..561c64ba3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -775,12 +775,14 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
endif()
target_include_directories(strata_dsv41_kernels PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories})
target_link_libraries(strata_dsv41_kernels PUBLIC ${_strata_gpu_runtime_target})
+ target_compile_options(strata_dsv41_kernels PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-include ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41/ds4_launch.h>)
add_executable(strata-dsv41 src/dsv41/ds4_main.cpp src/dsv41/ds4_model.cu src/dsv41/ds4_kernels.cu src/dsv41/ds4_vision.cu
src/dsv41/ds4_gguf.cu src/dsv41/ds4_gguf_cpu.cpp src/dsv41/ds4_iq.cu src/dsv41/ds4_iq_cpu.cpp
src/dsv41/ds4_tq.cu src/dsv41/ds4_tq_cpu.cpp src/dsv41/ds4_gg_cpu.cpp src/dsv41/ds4_packinfo.cpp)
target_include_directories(strata-dsv41 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${_strata_gpu_include_directories}
${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)
target_link_libraries(strata-dsv41 PRIVATE strata_dsv41_kernels ${_strata_gpu_runtime_target} ${_strata_gpu_blas_target})
+ target_compile_options(strata-dsv41 PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-include ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41/ds4_launch.h>)
find_package(Threads REQUIRED)
target_link_libraries(strata-dsv41 PRIVATE Threads::Threads)
if(WIN32 AND STRATA_ENABLE_HIP) # cudaMemGetInfo is the DXGI-budgeted hip_compat::mem_get_info (src/core/device.cu): on a Strix Halo the
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 1cc29a3f7..6678943ae 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -159,6 +159,17 @@ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {
d.tg[0] = 256;
d.tg[1] = d.tg[2] = 1;
g_cx->run({d});
+ // The fp8-rounding finishes (the KV rows, the attention output) are read stale by the next dispatch when it shares their
+ // command buffer: end the buffer after them (committed, not waited for: the queue orders the buffers).
+ static const bool no_split = std::getenv("DS4_NO_FINISH_SPLIT") != nullptr;
+ if (f.quant && !no_split) g_cx->split_runs();
+ if (const char* fm2 = std::getenv("DS4_FLUSH_FINISH_CTX")) { if (f.quant) { if (std::atoi(fm2) == 2) g_cx->split_runs(); else g_cx->flush_runs(); } }
+ if (const char* fm = std::getenv("DS4_FLUSH_FINISH")) { // debugging: which finish calls need a drain after them
+ const int m = std::atoi(fm);
+ if ((m == 1 && f.dst) || (m == 2 && f.rd > 0) || (m == 3 && f.w) || (m == 4 && f.quant) || (m == 5 && !f.dst) ||
+ (m == 6 && f.dst == nullptr && f.rd == 0) || (m == 7 && f.rd > 0 && f.inverse) || (m == 8 && f.rd > 0 && !f.inverse && !f.dst) || (m == 9 && f.rd > 0 && !f.dst) || (m == 10 && (f.dst || f.inverse)) || (m == 11 && f.rd > 0 && !f.inverse))
+ ds4_metal_flush();
+ }
return true;
}
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index e07af11f7..2bdae3bf9 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -181,7 +181,7 @@ kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[bu
if (e >= len) continue;
if (f.dst != nullptr) {
const uint bits = as_type<uint>(r[j]);
- f.dst[e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even
+ f.dst[(long) b * f.dst_stride + (long) e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even, the vector's own row
} else {
v[e] = r[j];
}
@@ -622,7 +622,7 @@ kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device uchar
const device ushort* src = ext + (size_t) (n + t) * d;
for (uint e0 = 0; e0 < d; e0 += 256u) {
const uint e = e0 + tid;
- const float v = e < d ? float(as_type<half>(src[e])) : 0.0f;
+ const float v = e < d ? as_type<float>(uint(src[e]) << 16) : 0.0f; // (BF16 bits, not half)
uint a = as_type<uint>(v) & 0x7fffffffu; // magnitudes ordered as bits
for (uint o = 16u; o > 0u; o >>= 1u) a = max(a, simd_shuffle_xor(a, o));
const int k = a == 0u ? 0 : int(a >> 23) - 135 + (((a & 0x7fffffu) > 0x600000u) ? 1 : 0);
diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cpp
index 70772bda8..0735ef955 100644
--- a/metal/src/gpu/runtime.cpp
+++ b/metal/src/gpu/runtime.cpp
@@ -322,6 +322,7 @@ Error free_alloc(void* p) {
size_t off = 0;
const Alloc* a = find(p, &off);
if (!a || off != 0) return fail(kErrorInvalidValue);
+ if (std::getenv("DS4_TRACE_FREE")) std::fprintf(stderr, "gpu free %zu bytes (host=%d) at %p\n", a->bytes, (int) a->host, p);
device_synchronize(); // as cudaFree: the GPU is done with it
std::lock_guard<std::mutex> lk(r.mu);
MTL::Buffer* b = a->buf;
@@ -349,6 +350,8 @@ void* buffer_for_cpu_impl(const void* cpu, size_t bytes, size_t* off) {
return (void*) a.buf;
}
+MTL::ResidencySet* residency_set_impl() { return rt().rs; }
+
void* host_view(const void* p) {
size_t off = 0;
const Alloc* a = find(p, &off);
@@ -703,3 +706,8 @@ Stats stats() { return rt().stats; }
extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) {
return strata::gpu::buffer_for_cpu_impl(cpu, bytes, off);
}
+
+// The residency set every strata::gpu stream queue carries: the Context's queue (mtl_rt) needs it too, because kernels
+// dereference raw gpuAddresses (weights, ring buffers, job arrays) that no setBuffer names, and memory that is not resident
+// reads as zeros.
+extern "C" void* ds4_gpu_residency_set() { return (void*) strata::gpu::residency_set_impl(); }
diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.h
index f09185c80..1d47c99ca 100644
--- a/metal/src/mtl_rt.h
+++ b/metal/src/mtl_rt.h
@@ -101,6 +101,7 @@ public:
// Encodes `ds` into ONE compute encoder (serial or concurrent dispatch type) and ONE command buffer, `reps`
// times over; commits and waits. Returns the GPU time (GPUEndTime - GPUStartTime) in seconds.
+ void split_runs(); // commit the batched dispatches as their own command buffer, without waiting
void flush_runs(); // commit and wait for the dispatches batched since the last flush
double run(const std::vector<Dispatch>& ds, int reps = 1, bool concurrent = false, bool barrier_between = false);
// A dependency graph in ONE concurrent encoder: a memory barrier only before dispatches with `barrier` set, so
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index 56e3319c6..3bfafe53f 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -33,10 +33,12 @@ std::string read_file(const std::string& path) {
return ss.str();
}
+extern "C" void* ds4_gpu_residency_set() __attribute__((weak_import));
struct Context::Impl {
id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate in
id<MTLComputeCommandEncoder> enc_def = nil;
int def_n = 0;
+ id<MTLCommandBuffer> inflight = nil; // the last committed-but-not-waited deferred buffer (the queue runs them in order)
std::vector<void*> graveyard; // buffers freed while the deferred command buffer still references them (it holds none)
std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoder
id<MTLDevice> dev;
@@ -64,6 +66,10 @@ Context::Context() : impl_(new Impl) {
std::exit(2);
}
impl_->q = [impl_->dev newCommandQueue];
+ if (ds4_gpu_residency_set && !std::getenv("DS4_NO_RESIDENCY")) {
+ id<MTLResidencySet> rs = (__bridge id<MTLResidencySet>)ds4_gpu_residency_set();
+ if (rs) [impl_->q addResidencySet:rs];
+ }
// env, then <exe dir>/metal/ (a component package), then the compiled-in source-tree path
ggml_common_path = strata_metal::resource("STRATA_GGML_COMMON", "ggml-common.h", STRATA_GGML_COMMON_DEFAULT);
kernel_dir = strata_metal::resource("STRATA_METAL_KERNELS", "kernels", STRATA_METAL_KERNELS_DEFAULT);
@@ -267,7 +273,7 @@ void Context::free(Buffer& b) {
// is still queued is read after it is gone (or after another alloc took its memory: the job arrays of iq_mv read the
// NEXT call's jobs, measured as rows of zeros and different tokens every run). Hold it until the buffer has completed.
std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
- const bool hold = impl_->cb_def != nil;
+ const bool hold = impl_->cb_def != nil || impl_->inflight != nil;
if (b.chunks) {
for (void* h : *b.chunks) {
if (hold) impl_->graveyard.push_back(h);
@@ -294,6 +300,8 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
if (a.buf) {
size_t o;
id<MTLBuffer> mb = BO(*a.buf, a.off, o);
+ static const bool chkbuf = std::getenv("DS4_CHECK_BUF") != nullptr;
+ if (chkbuf && o >= [mb length]) std::fprintf(stderr, "mtl: %s arg %zu: offset %zu past the buffer (%zu bytes; Buffer.bytes %zu bias %zu off %zu)\n", d.p->name.c_str(), i, o, (size_t) [mb length], a.buf->bytes, a.buf->bias, (size_t) a.off);
[enc setBuffer:mb offset:o atIndex:i];
}
else
@@ -326,21 +334,41 @@ static void ds4_report_counts() {
extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }
extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));
-void Context::flush_runs() {
- if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);
+void Context::split_runs() {
+ // Commit the batched dispatches as their own command buffer WITHOUT waiting: the queue runs buffers in order and makes each
+ // one's writes visible to the next. (Measured: a producer kernel's output was read stale by the next dispatch of the same
+ // buffer for two kernels in a layer; ending the buffer after them is enough, no wait needed.)
std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
- if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }
if (impl_->cb_def == nil) return;
++g_cb_count;
- { const double d = impl_->cb_def.GPUEndTime - impl_->cb_def.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0; }
[impl_->enc_def endEncoding];
[impl_->cb_def commit];
- [impl_->cb_def waitUntilCompleted];
- if (impl_->cb_def.status != MTLCommandBufferStatusCompleted)
- std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->cb_def.error localizedDescription] UTF8String]);
+ impl_->inflight = impl_->cb_def;
impl_->cb_def = nil;
impl_->enc_def = nil;
impl_->def_n = 0;
+}
+
+void Context::flush_runs() {
+ if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);
+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
+ if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }
+ if (impl_->cb_def != nil) {
+ ++g_cb_count;
+ [impl_->enc_def endEncoding];
+ [impl_->cb_def commit];
+ impl_->inflight = impl_->cb_def;
+ impl_->cb_def = nil;
+ impl_->enc_def = nil;
+ impl_->def_n = 0;
+ }
+ if (impl_->inflight != nil) {
+ [impl_->inflight waitUntilCompleted];
+ { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0; }
+ if (impl_->inflight.status != MTLCommandBufferStatusCompleted)
+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);
+ impl_->inflight = nil;
+ }
for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
impl_->graveyard.clear();
}
@@ -370,7 +398,9 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,
++impl_->def_n;
++g_disp_count;
static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr; // debugging: every dispatch alone, in order
- if (each) {
+ static const std::string only = std::getenv("DS4_FLUSH_AFTER") ? std::getenv("DS4_FLUSH_AFTER") : ""; // ",name1,name2,"
+ const bool listed = !only.empty() && only.find("," + ds[0].p->name + ",") != std::string::npos;
+ if (each || listed) {
if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();
flush_runs();
}
diff --git a/src/dsv41/ds4_launch.h b/src/dsv41/ds4_launch.h
new file mode 100644
index 000000000..7d0bc9ad8
--- /dev/null
+++ b/src/dsv41/ds4_launch.h
@@ -0,0 +1,28 @@
+// strata_ds4_launch(kernel, grid, block, shared, stream, args...): the engine's one spelling of a kernel launch. On CUDA it is
+// kernel<<<grid, block, shared, stream>>>(args...); on Metal metal/include/metal_compat/cuda_launch.h maps the same call onto
+// a dispatch. (Force-included for CUDA compiles by CMakeLists.txt.)
+#pragma once
+#if defined(__CUDACC__) && !defined(STRATA_METAL)
+#include <cuda_runtime.h>
+#include <cstdio>
+#include <cstdlib>
+template <class... KA, class... CA>
+inline void strata_ds4_launch_at(const char* file, int line, void (*kernel)(KA...), dim3 grid, dim3 block, size_t shared,
+ cudaStream_t stream, CA&&... args) {
+ kernel<<<grid, block, shared, stream>>>(static_cast<KA>(args)...);
+ static const bool check = std::getenv("DS4_LAUNCH_CHECK") != nullptr; // report the launch that failed (not a later call)
+ if (check) {
+ const cudaError_t e = cudaGetLastError();
+ if (e != cudaSuccess)
+ std::fprintf(stderr, "ds4 launch failed at %s:%d: %s (grid %u,%u,%u block %u,%u,%u shared %zu)\n", file, line,
+ cudaGetErrorString(e), grid.x, grid.y, grid.z, block.x, block.y, block.z, shared);
+ }
+}
+#define strata_ds4_launch(...) strata_ds4_launch_at(__FILE__, __LINE__, __VA_ARGS__)
+
+// (the Metal debug dumps in ds4_model.cu name these; on CUDA there is no deferred queue and a device pointer is not a host view)
+inline void ds4_metal_flush() {}
+namespace strata::gpu {
+inline void* host_view(const void* p) { return const_cast<void*>(p); }
+}
+#endif
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 240422119..0c51ea881 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -7780,6 +7780,8 @@ struct Model::Impl {
CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,
(size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));
engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);
+ dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);
+ dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);
if (std::getenv("DS4_DBG_EG")) {
dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);
const float* xv = (const float*) strata::gpu::host_view(d.eg_x);
@@ -7796,7 +7798,11 @@ struct Model::Impl {
}
if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;
if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);
+ dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);
+ dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);
engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);
+ dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);
if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);
}
dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);
@@ -7852,6 +7858,7 @@ struct Model::Impl {
finish(d.kvr, T, HD, HD, fk, s);
if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);
dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);
+ dump_sub(d, "ext", l.il, p0, T, d.ext, ((size_t) g.win + T) * HD * 2); // (BF16 rows: the ring's, then this window's)
AttnRows ar;
for (int t = 0; t < T; ++t) {
if (l.mtp) {