Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: the ring backup flag, decode takes the host-routed path, split commits sync the streams first, sparse_attn sizes its binding without a host read of device data

Niko1221860e384db78e
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 6678943ae..8ad2eea5f 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -19,6 +19,7 @@
// called by the gpu layer's synchronisation: everything batched must reach the GPU before a host read
extern "C" void ds4_metal_flush();
+extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu);
extern "C" void ds4_gpu_sync_all_streams();
extern "C" unsigned long long ds4_dispatch_seq();
static unsigned long long g_q8_deduped = 0;
@@ -956,7 +957,7 @@ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d,
dsp.p = &pl;
dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bb, b_off),
mtl::Arg::V((long) p0), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) T),
- mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};
+ mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes), mtl::Arg::V((unsigned) (hb ? 1 : 0))};
dsp.grid[0] = (uint32_t) T;
dsp.grid[1] = dsp.grid[2] = 1;
dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;
@@ -1021,14 +1022,10 @@ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int
if (ar.gen != 0 && ar.R > 0) {
crows = (size_t) ((ar.p0 + ar.t0 + T + 1) / ar.R) + 2; // every selected row is below this position
} else {
- int maxrow = -1; // the selected rows, not the cache's whole cap
- if (hs != nullptr && sel_stride > 0)
- for (int t = 0; t < T; ++t)
- for (int i = 0; i < ar.nsel[t]; ++i) {
- const int r = ((const int*) hs)[(size_t) t * (size_t) sel_stride + (size_t) i];
- if (r > maxrow) maxrow = r;
- }
- crows = (size_t) (maxrow + 1);
+ // (the rows the selection names are on the device and not read back here: an earlier build read them on the
+ // host while the kernel that wrote them was still queued, and sized the binding from stale zeros)
+ const size_t rem = ds4_gpu_alloc_remaining(hc);
+ crows = rem > 0 ? rem / 288 : (size_t) (ar.nsel[0] > 0 ? ar.nsel[0] : 1);
}
if (crows < 1) crows = 1;
} else {
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index 2bdae3bf9..665087cb7 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -611,11 +611,11 @@ static inline uint ds4_e4m3_code(float v, int k) {
kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device uchar* ring [[buffer(1)]],
device uchar* backup [[buffer(2)]], constant long& p0 [[buffer(3)]],
constant uint& n [[buffer(4)]], constant uint& T [[buffer(5)]],
- constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]],
+ constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]], constant uint& has_backup [[buffer(8)]],
uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {
if (T - t > n) return; // overwritten by a later row of the window anyway
device uchar* row = ring + (size_t) ((size_t(p0) + t) % n) * rb;
- if (backup != nullptr) {
+ if (has_backup != 0u) { // (a bound buffer is never a null pointer here: the host says whether there is a backup)
for (size_t i = tid; i < rb; i += 256u) backup[(size_t) t * rb + i] = row[i];
threadgroup_barrier(mem_flags::mem_threadgroup);
}
diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cpp
index 0735ef955..788dd7654 100644
--- a/metal/src/gpu/runtime.cpp
+++ b/metal/src/gpu/runtime.cpp
@@ -711,3 +711,11 @@ extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* o
// dereference raw gpuAddresses (weights, ring buffers, job arrays) that no setBuffer names, and memory that is not resident
// reads as zeros.
extern "C" void* ds4_gpu_residency_set() { return (void*) strata::gpu::residency_set_impl(); }
+
+// bytes from `cpu` to the end of the allocation that holds it (0: not an allocation)
+extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu) {
+ size_t off = 0;
+ void* b = strata::gpu::buffer_for_cpu_impl(cpu, 1, &off);
+ if (!b) return 0;
+ return ((MTL::Buffer*) b)->length() - off;
+}
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index 3bfafe53f..440d4fcd0 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -335,6 +335,9 @@ extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }
extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));
void Context::split_runs() {
+ // (the streams' pending copies come first, as in a full flush: they are queued on another MTLCommandQueue and would
+ // otherwise not have run when the commands committed here read what they write)
+ if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();
// Commit the batched dispatches as their own command buffer WITHOUT waiting: the queue runs buffers in order and makes each
// one's writes visible to the next. (Measured: a producer kernel's output was read stale by the next dispatch of the same
// buffer for two kernels in a layer; ending the buffer after them is enough, no wait needed.)
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 0c51ea881..bfdca034e 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -920,7 +920,13 @@ struct Model::Impl {
std::atomic<int> layer_of[kRing];
std::thread service;
std::atomic<bool> quit{false};
+#if defined(STRATA_METAL)
+ // (the async decode has the GPU wait on flags the CPU lane sets while it runs: the Metal queue cannot be waited on in the
+ // middle of a command buffer, so a decode step takes the host-routed path, as the prompt windows do)
+ bool sync_decode = std::getenv("DS4_ASYNC") == nullptr;
+#else
bool sync_decode = std::getenv("DS4_SYNC") != nullptr;
+#endif
// a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run
// before the wait for the CPU lane - the same bits, no faster)
bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;
@@ -1175,8 +1181,12 @@ struct Model::Impl {
if (x == il) return true;
return false;
}
+ bool dump_s_name(const char* name) const { // DS4_DUMP_S_NAMES=a,b,c: only these points (default all)
+ static const std::string names = [] { const char* e = std::getenv("DS4_DUMP_S_NAMES"); return std::string(e ? "," : "") + (e ? e : "") + (e ? "," : ""); }();
+ return names.empty() || names.find(std::string(",") + name + ",") != std::string::npos;
+ }
void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {
- if (!dump_s_on(il)) return;
+ if (!dump_s_on(il) || !dump_s_name(name)) return;
std::vector<uint8_t> v(nbytes);
cudaSetDevice(d.id);
#if defined(STRATA_METAL)
@@ -1187,7 +1197,7 @@ struct Model::Impl {
dump_sub_host(name, il, p0, T, v.data(), nbytes);
}
void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {
- if (!dump_s_on(il)) return;
+ if (!dump_s_on(il) || !dump_s_name(name)) return;
std::lock_guard<std::mutex> lk(dump_mu);
char nm[16] = {};
std::strncpy(nm, name, 15);
@@ -7744,6 +7754,7 @@ struct Model::Impl {
const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;
const ArGroup& ag = groups[(size_t) L0.grp].ar;
const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;
+ if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);
if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG
std::vector<float> hv((size_t) T * H * D);
cudaSetDevice(d.id);
@@ -7806,6 +7817,7 @@ struct Model::Impl {
if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);
}
dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);
+ dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));
// attention
hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);
hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);
@@ -7897,6 +7909,8 @@ struct Model::Impl {
CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));
}
compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);
+ dump_sub(d, "ckr", l.il, p0, T, d.ckr, (size_t) T * HD * 4);
+ dump_sub(d, "cgr", l.il, p0, T, d.cgr, (size_t) T * HD * 4);
}
}
Rows lim;
@@ -8001,6 +8015,8 @@ struct Model::Impl {
xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);
}
}
+ if (ng > 0) dump_sub(d, "latent", l.il, p0, ng, d.latent, (size_t) ng * HD * 4);
+ dump_sub(d, "topk", l.il, p0, T, d.topk, (size_t) T * topk_stride() * 4);
for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;
ckv = sr.ckv[(size_t) l.dev];
}