Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: copies and fills ride the context's command buffer, events mark it, buffers chained by a shared event, decode matvecs for IQ2_XS/IQ2_S/IQ3_XXS

Niko122142a982e03323
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 7da666011..af2eba254 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -584,7 +584,7 @@ bool iq_mv(const void* jobs, int nj) {
if (nj <= 0) return true;
if (!ensure()) return false;
const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;
- std::vector<Ds4IqMvJob> exp_iq, exp_k, exp_q5, exp_x1;
+ std::vector<Ds4IqMvJob> exp_iq, exp_k, exp_q5, exp_x1, exp_x2, exp_x3, exp_x4;
exp_iq.reserve((size_t) nj);
exp_k.reserve((size_t) nj);
int max_n = 0;
@@ -605,6 +605,9 @@ bool iq_mv(const void* jobs, int nj) {
static const bool fast1 = std::getenv("DS4_NO_MV1") == nullptr;
if (fast1 && b.ncol == 1 && b.type == 13) exp_q5.push_back(b); // the decode step's one-activation kernels
else if (fast1 && b.ncol == 1 && b.type == 16) exp_x1.push_back(b);
+ else if (fast1 && b.ncol == 1 && b.type == 17) exp_x2.push_back(b);
+ else if (fast1 && b.ncol == 1 && b.type == 22) exp_x3.push_back(b);
+ else if (fast1 && b.ncol == 1 && b.type == 18) exp_x4.push_back(b);
else (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);
}
}
@@ -643,6 +646,9 @@ bool iq_mv(const void* jobs, int nj) {
};
if (!run_group(exp_q5, "ds4_q5k_mv1")) return false;
if (!run_group(exp_x1, "ds4_iq2xxs_mv1")) return false;
+ if (!run_group(exp_x2, "ds4_iq2xs_mv1")) return false;
+ if (!run_group(exp_x3, "ds4_iq2s_mv1")) return false;
+ if (!run_group(exp_x4, "ds4_iq3xxs_mv1")) return false;
if (!run_group(exp_k, "ds4_iq_mv_k")) return false;
if (!run_group(exp_iq, "ds4_iq_mv")) return false;
return true;
@@ -2036,6 +2042,68 @@ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, voi
}
+
+// ---- copies, fills and event marks on the context's queue (see ds4_copy4 in ds4_init.metal) -------------------------------
+// strata::gpu routes cudaMemcpyAsync / cudaMemsetAsync here when both ends are its allocations; ds4_ctx_mark ends the batched
+// command buffer without waiting and returns a token the matching ds4_ctx_wait blocks on (cudaEventRecord / EventSynchronize).
+extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) {
+ if (n == 0) return true;
+ if (!ensure()) return false;
+ void* hd = strata::gpu::host_view(dst);
+ void* hs = strata::gpu::host_view((void*) src);
+ if (!hd || !hs) return false;
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) host) + bytes;
+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ mtl::Buffer bs = wrap(hs, n);
+ mtl::Buffer bd = wrap(hd, n);
+ if (!bs.handle || !bd.handle) return false;
+ const bool w4 = (((uintptr_t) hd | (uintptr_t) hs | n) & 3u) == 0;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, w4 ? "ds4_copy4" : "ds4_copy1"); } catch (...) { return false; }
+ const size_t items = w4 ? n / 4 : n;
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bd, off(hd)), mtl::Arg::V((unsigned long) items)};
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ d.grid[0] = (uint32_t) ((items + 255) / 256);
+ d.grid[1] = d.grid[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) {
+ if (n == 0) return true;
+ if (!ensure()) return false;
+ void* hd = strata::gpu::host_view(dst);
+ if (!hd) return false;
+ const size_t page = 16384;
+ const size_t base = (size_t) ((uintptr_t) hd) & ~(size_t) (page - 1);
+ const size_t end = (size_t) ((uintptr_t) hd) + n;
+ mtl::Buffer bd = g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);
+ if (!bd.handle) return false;
+ mtl::Pipeline pl;
+ try { pl = g_cx->pipeline(g_lib, "ds4_fill1"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(bd, (size_t) ((uintptr_t) hd - base)), mtl::Arg::V((unsigned long) n), mtl::Arg::V((unsigned) (value & 0xff))};
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ d.grid[0] = (uint32_t) ((n + 255) / 256);
+ d.grid[1] = d.grid[2] = 1;
+ g_cx->run({d});
+ return true;
+}
+extern "C" void* ds4_ctx_mark() {
+ if (!g_cx) return nullptr;
+ return g_cx->mark();
+}
+extern "C" void ds4_ctx_wait(void* token) {
+ if (g_cx && token) g_cx->wait_mark(token);
+}
+
extern "C" void ds4_metal_flush() {
// The engine's copies (the embedding rows, the streamed experts, the readbacks) are encoded on strata::gpu's default
// stream, its dispatched kernels on mtl_rt's deferred command buffer. Two queues, so committing one alone leaves the
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index 20c60ed24..db5af13ee 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -2058,3 +2058,19 @@ kernel void ds4_hc_split4(device const float* mixes [[buffer(0)]], device const
}
if (on) comb[(size_t) b * 64u + lane] = c;
}
+
+// Plain byte copies and fills on the context's queue: cudaMemcpyAsync / cudaMemsetAsync between runtime allocations (unified
+// memory: a "device" and a "host" buffer are both just memory) ride in the same command buffer as the kernels around them,
+// so their order is the program order and a layer needs no queue-to-queue handshake.
+kernel void ds4_copy4(device const uint* src [[buffer(0)]], device uint* dst [[buffer(1)]], constant ulong& n4 [[buffer(2)]],
+ uint gid [[thread_position_in_grid]]) {
+ if (ulong(gid) < n4) dst[gid] = src[gid];
+}
+kernel void ds4_copy1(device const uchar* src [[buffer(0)]], device uchar* dst [[buffer(1)]], constant ulong& n [[buffer(2)]],
+ uint gid [[thread_position_in_grid]]) {
+ if (ulong(gid) < n) dst[gid] = src[gid];
+}
+kernel void ds4_fill1(device uchar* dst [[buffer(0)]], constant ulong& n [[buffer(1)]], constant uint& v [[buffer(2)]],
+ uint gid [[thread_position_in_grid]]) {
+ if (ulong(gid) < n) dst[gid] = uchar(v);
+}
diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metal
index 493e5aff1..10d7500b4 100644
--- a/metal/src/engine/msl/ds4_iq.metal
+++ b/metal/src/engine/msl/ds4_iq.metal
@@ -746,3 +746,131 @@ kernel void ds4_iq2xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[t
const float v = simd_sum(acc);
if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
}
+
+// The other low-bit expert types, one activation (see ds4_iq2xxs_mv1): a lane a 32-value sub-block, four blocks a pass.
+
+kernel void ds4_iq2xs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u;
+ const uint bsel = lane >> 3;
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w + (size_t) blk * 74u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device ushort* qs = (const device ushort*) (p + 2u + 8u * ib);
+ const uint sc = uint(p[66u + ib]);
+ const float db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f, db1 = d * (0.5f + float(sc >> 4)) * 0.25f;
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int s0 = 0, s1 = 0;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint q = uint(qs[l]);
+ const ulong g = iq2xs_grid[q & 511u];
+ const uint sg = uint(ksigns_iq2xs[(q >> 9) & 127u]);
+ int sum = 0;
+ for (uint h = 0; h < 2u; ++h) {
+ const uint gw = uint(g >> (32u * h));
+ const uint x = xw[2u * l + h];
+ for (uint i = 0; i < 4u; ++i) {
+ const int code = int((gw >> (8u * i)) & 0xffu);
+ const int xv = int(char(x >> (8u * i)));
+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;
+ }
+ }
+ if (l < 2u) s0 += sum; else s1 += sum;
+ }
+ acc = fma(b.xd[blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+}
+
+kernel void ds4_iq2s_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u;
+ const uint bsel = lane >> 3;
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w + (size_t) blk * 82u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device uchar* qs = p + 2u + 4u * ib;
+ const device uchar* sgb = p + 34u + 4u * ib;
+ const uint qh = uint(p[66u + ib]);
+ const uint sc = uint(p[74u + ib]);
+ const float db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f, db1 = d * (0.5f + float(sc >> 4)) * 0.25f;
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int s0 = 0, s1 = 0;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint idx = uint(qs[l]) | ((qh << (8u - 2u * l)) & 0x300u);
+ const ulong g = iq2s_grid[idx];
+ const uint sg = uint(sgb[l]);
+ int sum = 0;
+ for (uint h = 0; h < 2u; ++h) {
+ const uint gw = uint(g >> (32u * h));
+ const uint x = xw[2u * l + h];
+ for (uint i = 0; i < 4u; ++i) {
+ const int code = int((gw >> (8u * i)) & 0xffu);
+ const int xv = int(char(x >> (8u * i)));
+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;
+ }
+ }
+ if (l < 2u) s0 += sum; else s1 += sum;
+ }
+ acc = fma(b.xd[blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+}
+
+kernel void ds4_iq3xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u;
+ const uint bsel = lane >> 3;
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w + (size_t) blk * 98u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device uchar* qs = p + 2u;
+ const uint w0 = uint(qs[8u * ib]) | (uint(qs[8u * ib + 1u]) << 8) | (uint(qs[8u * ib + 2u]) << 16) | (uint(qs[8u * ib + 3u]) << 24);
+ const uint w1 = uint(qs[8u * ib + 4u]) | (uint(qs[8u * ib + 5u]) << 8) | (uint(qs[8u * ib + 6u]) << 16) | (uint(qs[8u * ib + 7u]) << 24);
+ const uint aux = uint(qs[64u + 4u * ib]) | (uint(qs[65u + 4u * ib]) << 8) | (uint(qs[66u + 4u * ib]) << 16) | (uint(qs[67u + 4u * ib]) << 24);
+ const float db = d * (0.5f + float(aux >> 28)) * 0.5f;
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int sum = 0;
+ for (uint l = 0; l < 4u; ++l) {
+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);
+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);
+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];
+ const uint xl = xw[2u * l], xh = xw[2u * l + 1u];
+ for (uint i = 0; i < 4u; ++i) {
+ int v0 = int((g0 >> (8u * i)) & 0xffu), v1 = int((g1 >> (8u * i)) & 0xffu);
+ if ((sgn & (1u << i)) != 0u) v0 = -v0;
+ if ((sgn & (1u << (i + 4u))) != 0u) v1 = -v1;
+ sum += v0 * int(char(xl >> (8u * i))) + v1 * int(char(xh >> (8u * i)));
+ }
+ }
+ acc = fma(b.xd[blk * 8u + ib] * db, float(sum), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+}
diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cpp
index 788dd7654..cc8b054a7 100644
--- a/metal/src/gpu/runtime.cpp
+++ b/metal/src/gpu/runtime.cpp
@@ -68,6 +68,7 @@ struct Op {
struct EventImpl {
MTL::SharedEvent* ev = nullptr; // the recording stream's timeline
uint64_t value = 0; // 0: never recorded (complete, as a fresh CUDA event)
+ void* token = nullptr; // a mark on the context's queue (ds4_ctx_mark), released by the wait
};
struct GraphImpl {
@@ -195,6 +196,14 @@ void flush(StreamImpl* s) {
s->pending = 0;
}
extern "C" void ds4_metal_flush();
+extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) __attribute__((weak_import));
+extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) __attribute__((weak_import));
+extern "C" void* ds4_ctx_mark() __attribute__((weak_import));
+extern "C" void ds4_ctx_wait(void* token) __attribute__((weak_import));
+static bool ctx_queue_ops() { // DS4_STREAM_COPIES=1: the old way (a copy on a stream's own queue, a full barrier each)
+ static const bool on = ds4_ctx_copy != nullptr && std::getenv("DS4_STREAM_COPIES") == nullptr;
+ return on;
+}
Error wait_value(MTL::SharedEvent* ev, uint64_t v) {
while (ev->signaledValue() < v) std::this_thread::yield();
@@ -493,6 +502,10 @@ Error memset_device(void* p, int value, size_t bytes) {
return e;
}
Error memset_async(void* p, int value, size_t bytes, Stream st) {
+ if (!S(st)->rec && ctx_queue_ops() && bytes > 0) {
+ size_t o1 = 0;
+ if (find(p, &o1) && ds4_ctx_fill(p, value, bytes)) return kSuccess;
+ }
if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();
StreamImpl* s = S(st);
if (s->rec) {
@@ -524,6 +537,10 @@ Error memcpy_peer_async(void* dst, int dst_dev, const void* src, int src_dev, si
return memcpy_async(dst, src, bytes, kDeviceToDevice, st);
}
Error memcpy_async(void* dst, const void* src, size_t bytes, CopyKind, Stream st) {
+ if (!S(st)->rec && ctx_queue_ops() && bytes > 0) {
+ size_t o1 = 0, o2 = 0;
+ if (find(dst, &o1) && find(src, &o2) && ds4_ctx_copy(dst, src, bytes)) return kSuccess;
+ }
if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();
StreamImpl* s = S(st);
if (s->rec) {
@@ -582,6 +599,12 @@ Error launch_host_func(Stream st, HostFn fn, void* arg) {
Error event_create(Event* e, unsigned) { *e = new EventImpl; return kSuccess; }
Error event_destroy(Event e) { delete e; return kSuccess; }
Error event_record(Event e, Stream st) {
+ if (ctx_queue_ops() && !S(st)->rec) { // the work the event follows is in the context's command buffer: end it, wait later
+ e->token = ds4_ctx_mark();
+ e->ev = nullptr;
+ e->value = 0;
+ return kSuccess;
+ }
StreamImpl* s = S(st);
if (s->rec) {
Op o;
@@ -591,8 +614,19 @@ Error event_record(Event e, Stream st) {
}
return record_event(e, s);
}
-Error event_query(Event e) { return !e->ev || e->ev->signaledValue() >= e->value ? kSuccess : kNotReady; }
-Error event_synchronize(Event e) { return e->ev ? wait_value(e->ev, e->value) : kSuccess; }
+Error event_query(Event e) {
+ if (e->token) return kNotReady; // (a mark waits for its wait)
+ return !e->ev || e->ev->signaledValue() >= e->value ? kSuccess : kNotReady;
+}
+Error event_synchronize(Event e) {
+ if (e->token) {
+ void* t = e->token;
+ e->token = nullptr;
+ ds4_ctx_wait(t);
+ return kSuccess;
+ }
+ return e->ev ? wait_value(e->ev, e->value) : kSuccess;
+}
Error event_elapsed_ms(float* ms, Event, Event) { *ms = 0.0f; return fail(kErrorUnsupported); }
Error begin_capture(Stream st) {
diff --git a/metal/src/mtl_rt.h b/metal/src/mtl_rt.h
index 1d47c99ca..831d8d3a5 100644
--- a/metal/src/mtl_rt.h
+++ b/metal/src/mtl_rt.h
@@ -101,6 +101,8 @@ public:
// Encodes `ds` into ONE compute encoder (serial or concurrent dispatch type) and ONE command buffer, `reps`
// times over; commits and waits. Returns the GPU time (GPUEndTime - GPUStartTime) in seconds.
+ void* mark(); // split_runs() that returns a token for the buffer just committed (nullptr: nothing was queued)
+ void wait_mark(void* token); // block until that buffer has completed, then drop the token
void split_runs(); // commit the batched dispatches as their own command buffer, without waiting
void flush_runs(); // commit and wait for the dispatches batched since the last flush
double run(const std::vector<Dispatch>& ds, int reps = 1, bool concurrent = false, bool barrier_between = false);
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index d6c2070f5..69ac72851 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -6,6 +6,7 @@
#include "resource_path.h"
#include <chrono>
+#include <execinfo.h>
#include <algorithm>
#include <atomic>
#include <mutex>
@@ -38,6 +39,8 @@ struct Context::Impl {
id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate in
id<MTLComputeCommandEncoder> enc_def = nil;
int def_n = 0;
+ id<MTLSharedEvent> qev = nil; // consecutive deferred buffers are chained through it (untracked buffers: the queue alone does not order them)
+ uint64_t qev_v = 0;
id<MTLCommandBuffer> inflight = nil; // the last committed-but-not-waited deferred buffer (the queue runs them in order)
std::vector<void*> graveyard; // buffers freed while the deferred command buffer still references them (it holds none)
std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoder
@@ -318,6 +321,7 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
}
static double g_gpu_ms = 0.0; // GPU time of every committed command buffer, summed
+static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host time: waiting on command buffers; encoding dispatches
static std::mutex g_prof_mu;
static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)
static std::string g_prof_last;
@@ -327,6 +331,7 @@ static bool g_count_reg = false;
static void ds4_report_counts() {
std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,
(unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());
+ std::fprintf(stderr, "mtl: host waited %.1f ms for the GPU in %llu waits; encoding the dispatches took %.1f ms\n", g_wait_ns.load() / 1e6, (unsigned long long) g_wait_n.load(), g_enc_ns.load() / 1e6);
if (!g_prof.empty()) {
std::vector<std::pair<double, std::string>> pv;
double tot = 0;
@@ -359,6 +364,8 @@ void Context::split_runs() {
if (impl_->cb_def == nil) return;
++g_cb_count;
[impl_->enc_def endEncoding];
+ if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];
+ [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
[impl_->cb_def commit];
impl_->inflight = impl_->cb_def;
impl_->cb_def = nil;
@@ -366,6 +373,23 @@ void Context::split_runs() {
impl_->def_n = 0;
}
+void* Context::mark() {
+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
+ if (impl_->cb_def != nil) split_runs(); // (the streams' pending work goes first, inside)
+ if (impl_->inflight == nil) return nullptr;
+ return (__bridge_retained void*) impl_->inflight;
+}
+
+void Context::wait_mark(void* token) {
+ id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;
+ const auto tw0 = std::chrono::steady_clock::now();
+ [cb waitUntilCompleted];
+ g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();
+ ++g_wait_n;
+ if (cb.status != MTLCommandBufferStatusCompleted)
+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);
+}
+
void Context::flush_runs() {
if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);
std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
@@ -373,6 +397,8 @@ void Context::flush_runs() {
if (impl_->cb_def != nil) {
++g_cb_count;
[impl_->enc_def endEncoding];
+ if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];
+ [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
[impl_->cb_def commit];
impl_->inflight = impl_->cb_def;
impl_->cb_def = nil;
@@ -380,7 +406,11 @@ void Context::flush_runs() {
impl_->def_n = 0;
}
if (impl_->inflight != nil) {
+ if (std::getenv("DS4_TRACE_WAIT")) { void* fr[8]; const int nf = backtrace(fr, 8); std::fprintf(stderr, "WAIT flush_runs:\n"); backtrace_symbols_fd(fr + 1, nf - 1, 2); }
+ const auto tw0 = std::chrono::steady_clock::now();
[impl_->inflight waitUntilCompleted];
+ g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();
+ ++g_wait_n;
{ const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;
if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }
if (impl_->inflight.status != MTLCommandBufferStatusCompleted)
@@ -396,11 +426,14 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,
@autoreleasepool {
if (ds.size() == 1 && reps == 1 && !barrier_between) {
if (impl_->cb_def == nil || impl_->def_n >= 2000000) {
- flush_runs();
+ if (impl_->cb_def != nil) split_runs(); // (a buffer committed earlier may still be running: the queue keeps the order)
impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];
+ if (impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];
impl_->enc_def = [impl_->cb_def computeCommandEncoderWithDispatchType:std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];
}
+ const auto te0 = std::chrono::steady_clock::now();
encode(impl_->enc_def, ds[0]);
+ g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();
static const bool trace = std::getenv("DS4_TRACE_DISP") != nullptr;
if (trace) { std::lock_guard<std::mutex> lk(g_hist_mu); std::fprintf(stderr, "TRACE disp %s\n", ds[0].p->name.c_str()); }
// Every wrapper binds its own MTLBuffer over the same memory, so Metal's hazard tracking cannot see that one