Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: copies and fills ride the context's command buffer, events mark it, buffers chained by a shared event, decode matvecs for IQ2_XS/IQ2_S/IQ3_XXS
Niko122142a982e03323
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 7da666011..af2eba254 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -584,7 +584,7 @@ bool iq_mv(const void* jobs, int nj) {if (nj <= 0) return true;if (!ensure()) return false;const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;- std::vector<Ds4IqMvJob> exp_iq, exp_k, exp_q5, exp_x1;+ std::vector<Ds4IqMvJob> exp_iq, exp_k, exp_q5, exp_x1, exp_x2, exp_x3, exp_x4;exp_iq.reserve((size_t) nj);exp_k.reserve((size_t) nj);int max_n = 0;@@ -605,6 +605,9 @@ bool iq_mv(const void* jobs, int nj) {static const bool fast1 = std::getenv("DS4_NO_MV1") == nullptr;if (fast1 && b.ncol == 1 && b.type == 13) exp_q5.push_back(b); // the decode step's one-activation kernelselse if (fast1 && b.ncol == 1 && b.type == 16) exp_x1.push_back(b);+ else if (fast1 && b.ncol == 1 && b.type == 17) exp_x2.push_back(b);+ else if (fast1 && b.ncol == 1 && b.type == 22) exp_x3.push_back(b);+ else if (fast1 && b.ncol == 1 && b.type == 18) exp_x4.push_back(b);else (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);}}@@ -643,6 +646,9 @@ bool iq_mv(const void* jobs, int nj) {};if (!run_group(exp_q5, "ds4_q5k_mv1")) return false;if (!run_group(exp_x1, "ds4_iq2xxs_mv1")) return false;+ if (!run_group(exp_x2, "ds4_iq2xs_mv1")) return false;+ if (!run_group(exp_x3, "ds4_iq2s_mv1")) return false;+ if (!run_group(exp_x4, "ds4_iq3xxs_mv1")) return false;if (!run_group(exp_k, "ds4_iq_mv_k")) return false;if (!run_group(exp_iq, "ds4_iq_mv")) return false;return true;@@ -2036,6 +2042,68 @@ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, voi}++// ---- copies, fills and event marks on the context's queue (see ds4_copy4 in ds4_init.metal) -------------------------------+// strata::gpu routes cudaMemcpyAsync / cudaMemsetAsync here when both ends are its allocations; ds4_ctx_mark ends the batched+// command buffer without waiting and returns a token the matching ds4_ctx_wait blocks on (cudaEventRecord / EventSynchronize).+extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) {+ if (n == 0) return true;+ if (!ensure()) return false;+ void* hd = strata::gpu::host_view(dst);+ void* hs = strata::gpu::host_view((void*) src);+ if (!hd || !hs) return false;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bs = wrap(hs, n);+ mtl::Buffer bd = wrap(hd, n);+ if (!bs.handle || !bd.handle) return false;+ const bool w4 = (((uintptr_t) hd | (uintptr_t) hs | n) & 3u) == 0;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, w4 ? "ds4_copy4" : "ds4_copy1"); } catch (...) { return false; }+ const size_t items = w4 ? n / 4 : n;+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bd, off(hd)), mtl::Arg::V((unsigned long) items)};+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ d.grid[0] = (uint32_t) ((items + 255) / 256);+ d.grid[1] = d.grid[2] = 1;+ g_cx->run({d});+ return true;+}+extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) {+ if (n == 0) return true;+ if (!ensure()) return false;+ void* hd = strata::gpu::host_view(dst);+ if (!hd) return false;+ const size_t page = 16384;+ const size_t base = (size_t) ((uintptr_t) hd) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) hd) + n;+ mtl::Buffer bd = g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ if (!bd.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_fill1"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bd, (size_t) ((uintptr_t) hd - base)), mtl::Arg::V((unsigned long) n), mtl::Arg::V((unsigned) (value & 0xff))};+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ d.grid[0] = (uint32_t) ((n + 255) / 256);+ d.grid[1] = d.grid[2] = 1;+ g_cx->run({d});+ return true;+}+extern "C" void* ds4_ctx_mark() {+ if (!g_cx) return nullptr;+ return g_cx->mark();+}+extern "C" void ds4_ctx_wait(void* token) {+ if (g_cx && token) g_cx->wait_mark(token);+}+extern "C" void ds4_metal_flush() {// The engine's copies (the embedding rows, the streamed experts, the readbacks) are encoded on strata::gpu's default// stream, its dispatched kernels on mtl_rt's deferred command buffer. Two queues, so committing one alone leaves thediff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex 20c60ed24..db5af13ee 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -2058,3 +2058,19 @@ kernel void ds4_hc_split4(device const float* mixes [[buffer(0)]], device const}if (on) comb[(size_t) b * 64u + lane] = c;}++// Plain byte copies and fills on the context's queue: cudaMemcpyAsync / cudaMemsetAsync between runtime allocations (unified+// memory: a "device" and a "host" buffer are both just memory) ride in the same command buffer as the kernels around them,+// so their order is the program order and a layer needs no queue-to-queue handshake.+kernel void ds4_copy4(device const uint* src [[buffer(0)]], device uint* dst [[buffer(1)]], constant ulong& n4 [[buffer(2)]],+ uint gid [[thread_position_in_grid]]) {+ if (ulong(gid) < n4) dst[gid] = src[gid];+}+kernel void ds4_copy1(device const uchar* src [[buffer(0)]], device uchar* dst [[buffer(1)]], constant ulong& n [[buffer(2)]],+ uint gid [[thread_position_in_grid]]) {+ if (ulong(gid) < n) dst[gid] = src[gid];+}+kernel void ds4_fill1(device uchar* dst [[buffer(0)]], constant ulong& n [[buffer(1)]], constant uint& v [[buffer(2)]],+ uint gid [[thread_position_in_grid]]) {+ if (ulong(gid) < n) dst[gid] = uchar(v);+}diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metalindex 493e5aff1..10d7500b4 100644--- a/metal/src/engine/msl/ds4_iq.metal+++ b/metal/src/engine/msl/ds4_iq.metal@@ -746,3 +746,131 @@ kernel void ds4_iq2xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[tconst float v = simd_sum(acc);if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);}++// The other low-bit expert types, one activation (see ds4_iq2xxs_mv1): a lane a 32-value sub-block, four blocks a pass.++kernel void ds4_iq2xs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u;+ const uint bsel = lane >> 3;+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w + (size_t) blk * 74u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device ushort* qs = (const device ushort*) (p + 2u + 8u * ib);+ const uint sc = uint(p[66u + ib]);+ const float db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f, db1 = d * (0.5f + float(sc >> 4)) * 0.25f;+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int s0 = 0, s1 = 0;+ for (uint l = 0; l < 4u; ++l) {+ const uint q = uint(qs[l]);+ const ulong g = iq2xs_grid[q & 511u];+ const uint sg = uint(ksigns_iq2xs[(q >> 9) & 127u]);+ int sum = 0;+ for (uint h = 0; h < 2u; ++h) {+ const uint gw = uint(g >> (32u * h));+ const uint x = xw[2u * l + h];+ for (uint i = 0; i < 4u; ++i) {+ const int code = int((gw >> (8u * i)) & 0xffu);+ const int xv = int(char(x >> (8u * i)));+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;+ }+ }+ if (l < 2u) s0 += sum; else s1 += sum;+ }+ acc = fma(b.xd[blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+}++kernel void ds4_iq2s_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u;+ const uint bsel = lane >> 3;+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w + (size_t) blk * 82u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device uchar* qs = p + 2u + 4u * ib;+ const device uchar* sgb = p + 34u + 4u * ib;+ const uint qh = uint(p[66u + ib]);+ const uint sc = uint(p[74u + ib]);+ const float db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f, db1 = d * (0.5f + float(sc >> 4)) * 0.25f;+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int s0 = 0, s1 = 0;+ for (uint l = 0; l < 4u; ++l) {+ const uint idx = uint(qs[l]) | ((qh << (8u - 2u * l)) & 0x300u);+ const ulong g = iq2s_grid[idx];+ const uint sg = uint(sgb[l]);+ int sum = 0;+ for (uint h = 0; h < 2u; ++h) {+ const uint gw = uint(g >> (32u * h));+ const uint x = xw[2u * l + h];+ for (uint i = 0; i < 4u; ++i) {+ const int code = int((gw >> (8u * i)) & 0xffu);+ const int xv = int(char(x >> (8u * i)));+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;+ }+ }+ if (l < 2u) s0 += sum; else s1 += sum;+ }+ acc = fma(b.xd[blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+}++kernel void ds4_iq3xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u;+ const uint bsel = lane >> 3;+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w + (size_t) blk * 98u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device uchar* qs = p + 2u;+ const uint w0 = uint(qs[8u * ib]) | (uint(qs[8u * ib + 1u]) << 8) | (uint(qs[8u * ib + 2u]) << 16) | (uint(qs[8u * ib + 3u]) << 24);+ const uint w1 = uint(qs[8u * ib + 4u]) | (uint(qs[8u * ib + 5u]) << 8) | (uint(qs[8u * ib + 6u]) << 16) | (uint(qs[8u * ib + 7u]) << 24);+ const uint aux = uint(qs[64u + 4u * ib]) | (uint(qs[65u + 4u * ib]) << 8) | (uint(qs[66u + 4u * ib]) << 16) | (uint(qs[67u + 4u * ib]) << 24);+ const float db = d * (0.5f + float(aux >> 28)) * 0.5f;+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int sum = 0;+ for (uint l = 0; l < 4u; ++l) {+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];+ const uint xl = xw[2u * l], xh = xw[2u * l + 1u];+ for (uint i = 0; i < 4u; ++i) {+ int v0 = int((g0 >> (8u * i)) & 0xffu), v1 = int((g1 >> (8u * i)) & 0xffu);+ if ((sgn & (1u << i)) != 0u) v0 = -v0;+ if ((sgn & (1u << (i + 4u))) != 0u) v1 = -v1;+ sum += v0 * int(char(xl >> (8u * i))) + v1 * int(char(xh >> (8u * i)));+ }+ }+ acc = fma(b.xd[blk * 8u + ib] * db, float(sum), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+}diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex 788dd7654..cc8b054a7 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -68,6 +68,7 @@ struct Op {struct EventImpl {MTL::SharedEvent* ev = nullptr; // the recording stream's timelineuint64_t value = 0; // 0: never recorded (complete, as a fresh CUDA event)+ void* token = nullptr; // a mark on the context's queue (ds4_ctx_mark), released by the wait};struct GraphImpl {@@ -195,6 +196,14 @@ void flush(StreamImpl* s) {s->pending = 0;}extern "C" void ds4_metal_flush();+extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) __attribute__((weak_import));+extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) __attribute__((weak_import));+extern "C" void* ds4_ctx_mark() __attribute__((weak_import));+extern "C" void ds4_ctx_wait(void* token) __attribute__((weak_import));+static bool ctx_queue_ops() { // DS4_STREAM_COPIES=1: the old way (a copy on a stream's own queue, a full barrier each)+ static const bool on = ds4_ctx_copy != nullptr && std::getenv("DS4_STREAM_COPIES") == nullptr;+ return on;+}Error wait_value(MTL::SharedEvent* ev, uint64_t v) {while (ev->signaledValue() < v) std::this_thread::yield();@@ -493,6 +502,10 @@ Error memset_device(void* p, int value, size_t bytes) {return e;}Error memset_async(void* p, int value, size_t bytes, Stream st) {+ if (!S(st)->rec && ctx_queue_ops() && bytes > 0) {+ size_t o1 = 0;+ if (find(p, &o1) && ds4_ctx_fill(p, value, bytes)) return kSuccess;+ }if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();StreamImpl* s = S(st);if (s->rec) {@@ -524,6 +537,10 @@ Error memcpy_peer_async(void* dst, int dst_dev, const void* src, int src_dev, sireturn memcpy_async(dst, src, bytes, kDeviceToDevice, st);}Error memcpy_async(void* dst, const void* src, size_t bytes, CopyKind, Stream st) {+ if (!S(st)->rec && ctx_queue_ops() && bytes > 0) {+ size_t o1 = 0, o2 = 0;+ if (find(dst, &o1) && find(src, &o2) && ds4_ctx_copy(dst, src, bytes)) return kSuccess;+ }if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();StreamImpl* s = S(st);if (s->rec) {@@ -582,6 +599,12 @@ Error launch_host_func(Stream st, HostFn fn, void* arg) {Error event_create(Event* e, unsigned) { *e = new EventImpl; return kSuccess; }Error event_destroy(Event e) { delete e; return kSuccess; }Error event_record(Event e, Stream st) {+ if (ctx_queue_ops() && !S(st)->rec) { // the work the event follows is in the context's command buffer: end it, wait later+ e->token = ds4_ctx_mark();+ e->ev = nullptr;+ e->value = 0;+ return kSuccess;+ }StreamImpl* s = S(st);if (s->rec) {Op o;@@ -591,8 +614,19 @@ Error event_record(Event e, Stream st) {}return record_event(e, s);}-Error event_query(Event e) { return !e->ev || e->ev->signaledValue() >= e->value ? kSuccess : kNotReady; }-Error event_synchronize(Event e) { return e->ev ? wait_value(e->ev, e->value) : kSuccess; }+Error event_query(Event e) {+ if (e->token) return kNotReady; // (a mark waits for its wait)+ return !e->ev || e->ev->signaledValue() >= e->value ? kSuccess : kNotReady;+}+Error event_synchronize(Event e) {+ if (e->token) {+ void* t = e->token;+ e->token = nullptr;+ ds4_ctx_wait(t);+ return kSuccess;+ }+ return e->ev ? wait_value(e->ev, e->value) : kSuccess;+}Error event_elapsed_ms(float* ms, Event, Event) { *ms = 0.0f; return fail(kErrorUnsupported); }Error begin_capture(Stream st) {diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.hindex 1d47c99ca..831d8d3a5 100644--- a/metal/src/mtl_rt.h+++ b/metal/src/mtl_rt.h@@ -101,6 +101,8 @@ public:// Encodes `ds` into ONE compute encoder (serial or concurrent dispatch type) and ONE command buffer, `reps`// times over; commits and waits. Returns the GPU time (GPUEndTime - GPUStartTime) in seconds.+ void* mark(); // split_runs() that returns a token for the buffer just committed (nullptr: nothing was queued)+ void wait_mark(void* token); // block until that buffer has completed, then drop the tokenvoid split_runs(); // commit the batched dispatches as their own command buffer, without waitingvoid flush_runs(); // commit and wait for the dispatches batched since the last flushdouble run(const std::vector<Dispatch>& ds, int reps = 1, bool concurrent = false, bool barrier_between = false);diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex d6c2070f5..69ac72851 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -6,6 +6,7 @@#include "resource_path.h"#include <chrono>+#include <execinfo.h>#include <algorithm>#include <atomic>#include <mutex>@@ -38,6 +39,8 @@ struct Context::Impl {id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate inid<MTLComputeCommandEncoder> enc_def = nil;int def_n = 0;+ id<MTLSharedEvent> qev = nil; // consecutive deferred buffers are chained through it (untracked buffers: the queue alone does not order them)+ uint64_t qev_v = 0;id<MTLCommandBuffer> inflight = nil; // the last committed-but-not-waited deferred buffer (the queue runs them in order)std::vector<void*> graveyard; // buffers freed while the deferred command buffer still references them (it holds none)std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoder@@ -318,6 +321,7 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {}static double g_gpu_ms = 0.0; // GPU time of every committed command buffer, summed+static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host time: waiting on command buffers; encoding dispatchesstatic std::mutex g_prof_mu;static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)static std::string g_prof_last;@@ -327,6 +331,7 @@ static bool g_count_reg = false;static void ds4_report_counts() {std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,(unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());+ std::fprintf(stderr, "mtl: host waited %.1f ms for the GPU in %llu waits; encoding the dispatches took %.1f ms\n", g_wait_ns.load() / 1e6, (unsigned long long) g_wait_n.load(), g_enc_ns.load() / 1e6);if (!g_prof.empty()) {std::vector<std::pair<double, std::string>> pv;double tot = 0;@@ -359,6 +364,8 @@ void Context::split_runs() {if (impl_->cb_def == nil) return;++g_cb_count;[impl_->enc_def endEncoding];+ if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];+ [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];[impl_->cb_def commit];impl_->inflight = impl_->cb_def;impl_->cb_def = nil;@@ -366,6 +373,23 @@ void Context::split_runs() {impl_->def_n = 0;}+void* Context::mark() {+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);+ if (impl_->cb_def != nil) split_runs(); // (the streams' pending work goes first, inside)+ if (impl_->inflight == nil) return nullptr;+ return (__bridge_retained void*) impl_->inflight;+}++void Context::wait_mark(void* token) {+ id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;+ const auto tw0 = std::chrono::steady_clock::now();+ [cb waitUntilCompleted];+ g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();+ ++g_wait_n;+ if (cb.status != MTLCommandBufferStatusCompleted)+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);+}+void Context::flush_runs() {if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);std::lock_guard<std::recursive_mutex> lk_(impl_->mu);@@ -373,6 +397,8 @@ void Context::flush_runs() {if (impl_->cb_def != nil) {++g_cb_count;[impl_->enc_def endEncoding];+ if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];+ [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];[impl_->cb_def commit];impl_->inflight = impl_->cb_def;impl_->cb_def = nil;@@ -380,7 +406,11 @@ void Context::flush_runs() {impl_->def_n = 0;}if (impl_->inflight != nil) {+ if (std::getenv("DS4_TRACE_WAIT")) { void* fr[8]; const int nf = backtrace(fr, 8); std::fprintf(stderr, "WAIT flush_runs:\n"); backtrace_symbols_fd(fr + 1, nf - 1, 2); }+ const auto tw0 = std::chrono::steady_clock::now();[impl_->inflight waitUntilCompleted];+ g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();+ ++g_wait_n;{ const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }if (impl_->inflight.status != MTLCommandBufferStatusCompleted)@@ -396,11 +426,14 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,@autoreleasepool {if (ds.size() == 1 && reps == 1 && !barrier_between) {if (impl_->cb_def == nil || impl_->def_n >= 2000000) {- flush_runs();+ if (impl_->cb_def != nil) split_runs(); // (a buffer committed earlier may still be running: the queue keeps the order)impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];+ if (impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];}+ const auto te0 = std::chrono::steady_clock::now();encode(impl_->enc_def, ds[0]);+ g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();static const bool trace = std::getenv("DS4_TRACE_DISP") != nullptr;if (trace) { std::lock_guard<std::mutex> lk(g_hist_mu); std::fprintf(stderr, "TRACE disp %s\n", ds[0].p->name.c_str()); }// Every wrapper binds its own MTLBuffer over the same memory, so Metal's hazard tracking cannot see that one