Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal runtime: the DeepSeek queue/residency/barrier rules sit behind STRATA_DS4_RUNTIME; strata_ds4 compiles its own copy of mtl_rt.mm and gpu/runtime.cpp, the Qwen engine keeps the 0.2.7 code
Niko12216a51355051cf
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txtindex 46f048e90..b8ac13f57 100644--- a/metal/CMakeLists.txt+++ b/metal/CMakeLists.txt@@ -143,7 +143,21 @@ list(APPEND STRATA_DS4_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/engine/exl3_legac# The CPU lane for the experts the GPU does not hold (ggml-cpu's NEON row dots): without it every non-resident expert contributes# zero and the residual stream stays the embedding (measured: layers 0-3 changed nothing, then random tokens).set_source_files_properties(${DS4}/ds4_gg_cpu.cpp PROPERTIES COMPILE_DEFINITIONS "STRATA_DSV41_TQ_CPU=1")-add_executable(strata_ds4 ${STRATA_DS4_SOURCES})+# The DeepSeek engine's queue, residency and barrier rules (deferred command buffers, the second queue, the residency set shared with+# strata::gpu, ...) live in mtl_rt.mm and gpu/runtime.cpp behind STRATA_DS4_RUNTIME. strata_ds4 compiles its own copy of these two+# files with it; the Qwen engine (strata) and every other target keep the code of 0.2.7 (the macro is off there, so the files read+# exactly as they did). The copies are linked as objects, so they win over the same symbols in the libraries.+add_library(ds4_rt_mm OBJECT src/mtl_rt.mm)+target_include_directories(ds4_rt_mm PRIVATE src ${STRATA_ROOT}/third_party/ggml ${STRATA_ROOT}/include)+target_compile_options(ds4_rt_mm PRIVATE $<$<COMPILE_LANGUAGE:OBJCXX>:-fobjc-arc>)+target_compile_definitions(ds4_rt_mm PRIVATE STRATA_DS4_RUNTIME=1 STRATA_GGML_COMMON_DEFAULT="${GGML_COMMON}"+ STRATA_METAL_KERNELS_DEFAULT="${CMAKE_CURRENT_SOURCE_DIR}/kernels")+add_library(ds4_rt_gpu OBJECT src/gpu/runtime.cpp)+target_include_directories(ds4_rt_gpu BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat+ ${CMAKE_CURRENT_SOURCE_DIR}/include ${STRATA_ROOT}/include)+target_include_directories(ds4_rt_gpu SYSTEM PRIVATE ${METAL_CPP_DIR})+target_compile_definitions(ds4_rt_gpu PRIVATE STRATA_DS4_RUNTIME=1 STRATA_GPU_METAL=1)+add_executable(strata_ds4 ${STRATA_DS4_SOURCES} $<TARGET_OBJECTS:ds4_rt_mm> $<TARGET_OBJECTS:ds4_rt_gpu>)target_include_directories(strata_ds4 PRIVATE${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat ${STRATA_ROOT}/include ${DS4} ${CMAKE_CURRENT_SOURCE_DIR}/src${STRATA_ROOT}/third_party/ggml ${STRATA_GGML_DIR}/ggml/include ${STRATA_GGML_DIR}/ggml/src ${STRATA_GGML_DIR}/ggml/src/ggml-cpu)diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cppindex cc8b054a7..f0547effa 100644--- a/metal/src/gpu/runtime.cpp+++ b/metal/src/gpu/runtime.cpp@@ -24,13 +24,21 @@#include <atomic>#include <chrono>#include <cstdio>+#if defined(STRATA_DS4_RUNTIME)#include <cstdlib>+#endif#include <cstring>#include <iterator>#include <map>+#if defined(STRATA_DS4_RUNTIME)#include <mutex> // recursive_mutex+#else+#include <mutex>+#endif#include <thread>+#if defined(STRATA_DS4_RUNTIME)#include <algorithm>+#endif#include <vector>namespace strata::gpu {@@ -68,7 +76,9 @@ struct Op {struct EventImpl {MTL::SharedEvent* ev = nullptr; // the recording stream's timelineuint64_t value = 0; // 0: never recorded (complete, as a fresh CUDA event)+#if defined(STRATA_DS4_RUNTIME)void* token = nullptr; // a mark on the context's queue (ds4_ctx_mark), released by the wait+#endif};struct GraphImpl {@@ -121,12 +131,16 @@ Error fail(Error e) {return e;}+#if defined(STRATA_DS4_RUNTIME)static std::vector<StreamImpl*> g_all_streams; // every stream's queue holds pending copies until it is committedstatic std::mutex g_all_streams_mu;+#endifStreamImpl* make_stream() {auto* s = new StreamImpl;+#if defined(STRATA_DS4_RUNTIME){ std::lock_guard<std::mutex> lk(g_all_streams_mu); g_all_streams.push_back(s); }+#endifs->q = rt().dev->newCommandQueue();s->q->addResidencySet(rt().rs);s->tl = rt().dev->newSharedEvent();@@ -169,32 +183,47 @@ MTL::CommandBuffer* open_cb(StreamImpl* s) {}return s->cb;}+#if defined(STRATA_DS4_RUNTIME)// StreamImpl's encoder/command-buffer state is touched from the loader, the prefetcher and the compute path.// Without this lock two encoders can be open on one command buffer at once, which Metal asserts on.static std::recursive_mutex g_stream_mu; // encode_copy/flush nest+#endifvoid end_encoder(StreamImpl* s) {+#if defined(STRATA_DS4_RUNTIME)if (s->enc) { if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] close compute enc %p\n", (void*) s->enc);s->enc->endEncoding(); s->enc = nullptr; }+#else+ if (s->enc) { s->enc->endEncoding(); s->enc = nullptr; }+#endif}MTL::ComputeCommandEncoder* encoder(StreamImpl* s) {+#if defined(STRATA_DS4_RUNTIME)std::lock_guard<std::recursive_mutex> lk(g_stream_mu);if (!s->enc) { s->enc = open_cb(s)->computeCommandEncoder();if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] open compute enc %p (cb %p)\n", (void*) s->enc, (void*) s->cb); } // serial dispatch: each dispatch sees the last+#else+ if (!s->enc) s->enc = open_cb(s)->computeCommandEncoder(); // serial dispatch: each dispatch sees the last+#endifreturn s->enc;}void flush(StreamImpl* s) {+#if defined(STRATA_DS4_RUNTIME)std::lock_guard<std::recursive_mutex> lk(g_stream_mu);+#endifif (!s->cb) return;end_encoder(s);+#if defined(STRATA_DS4_RUNTIME)if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] signal %llu on cb %p (enc %p)\n",(unsigned long long) (s->submitted + 1), (void*) s->cb, (void*) s->enc);+#endifs->cb->encodeSignalEvent(s->tl, ++s->submitted);s->cb->commit();s->cb->release();s->cb = nullptr;s->pending = 0;}+#if defined(STRATA_DS4_RUNTIME)extern "C" void ds4_metal_flush();extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) __attribute__((weak_import));extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) __attribute__((weak_import));@@ -205,6 +234,7 @@ static bool ctx_queue_ops() { // DS4_STREAM_COPIES=1: the old way (a copy on areturn on;}+#endifError wait_value(MTL::SharedEvent* ev, uint64_t v) {while (ev->signaledValue() < v) std::this_thread::yield();++rt().stats.host_syncs;@@ -222,7 +252,9 @@ void check_purged(const Alloc* a, const char* what) {}Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {+#if defined(STRATA_DS4_RUNTIME)std::lock_guard<std::recursive_mutex> lk(g_stream_mu);+#endifsize_t od = 0, os = 0;const Alloc* ad = find(dst, &od);const Alloc* as = find(src, &os);@@ -231,7 +263,9 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {if (ad && as) { // buffer to buffer: a blit, in stream orderend_encoder(s);MTL::BlitCommandEncoder* b = open_cb(s)->blitCommandEncoder();+#if defined(STRATA_DS4_RUNTIME)if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] blit copy %zu\n", n);+#endifb->copyFromBuffer(as->buf, os, ad->buf, od, n);b->endEncoding();++rt().stats.blits;@@ -248,7 +282,9 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {}Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {+#if defined(STRATA_DS4_RUNTIME)std::lock_guard<std::recursive_mutex> lk(g_stream_mu);+#endifsize_t off = 0;const Alloc* a = find(p, &off);if (!a) return fail(kErrorInvalidValue);@@ -262,7 +298,9 @@ Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {}Error encode_dispatch(StreamImpl* s, const Op& o) {+#if defined(STRATA_DS4_RUNTIME)std::lock_guard<std::recursive_mutex> lk(g_stream_mu);+#endifMTL::ComputeCommandEncoder* e = encoder(s);e->setComputePipelineState(o.pipe);if (o.big_args.empty()) e->setBytes(o.args, o.arg_bytes, 0);@@ -308,8 +346,12 @@ void alloc_log(const char* what, size_t n) {Error new_alloc(void** p, size_t bytes, bool host) {Runtime& r = rt();+#if defined(STRATA_DS4_RUNTIME)const size_t pg = 16384, want = bytes ? bytes : 1;const size_t n = std::getenv("DS4_NO_PAD") ? want : ((want + pg - 1) / pg) * pg;+#else+ const size_t n = bytes ? bytes : 1;+#endifMTL::Buffer* b = r.dev->newBuffer(n, MTL::ResourceStorageModeShared | MTL::ResourceHazardTrackingModeUntracked);if (!b) return fail(kErrorMemoryAllocation);Alloc a{b, b->gpuAddress(), (uint8_t*) b->contents(), n, host};@@ -331,7 +373,9 @@ Error free_alloc(void* p) {size_t off = 0;const Alloc* a = find(p, &off);if (!a || off != 0) return fail(kErrorInvalidValue);+#if defined(STRATA_DS4_RUNTIME)if (std::getenv("DS4_TRACE_FREE")) std::fprintf(stderr, "gpu free %zu bytes (host=%d) at %p\n", a->bytes, (int) a->host, p);+#endifdevice_synchronize(); // as cudaFree: the GPU is done with itstd::lock_guard<std::mutex> lk(r.mu);MTL::Buffer* b = a->buf;@@ -347,6 +391,7 @@ Error free_alloc(void* p) {} // namespace// ---------------------------------------------------------------------------------------------------- runtime.hpp+#if defined(STRATA_DS4_RUNTIME)void* buffer_for_cpu_impl(const void* cpu, size_t bytes, size_t* off) {Runtime& r = rt();std::lock_guard<std::mutex> lk(r.mu);@@ -361,6 +406,7 @@ void* buffer_for_cpu_impl(const void* cpu, size_t bytes, size_t* off) {MTL::ResidencySet* residency_set_impl() { return rt().rs; }+#endifvoid* host_view(const void* p) {size_t off = 0;const Alloc* a = find(p, &off);@@ -405,7 +451,9 @@ Error get_device(int* dev) { *dev = 0; return kSuccess; }Error set_device(int dev) { return dev == 0 ? kSuccess : fail(kErrorInvalidValue); }Error device_count(int* n) { *n = rt().dev ? 1 : 0; return kSuccess; }Error device_synchronize() {+#if defined(STRATA_DS4_RUNTIME)ds4_metal_flush();+#endifRuntime& r = rt();StreamImpl* s = S(nullptr);flush(s);@@ -475,6 +523,7 @@ Error host_device_pointer_raw(void** dev, void* host) {*dev = (void*) (uintptr_t) (a->gpu + off);return kSuccess;}+#if defined(STRATA_DS4_RUNTIME)// A synchronising call is a barrier for BOTH queues: the dispatches deferred in mtl_rt's command buffer have to run before// the host reads or overwrites what they touch (a window's last layers still pending while the next window's embedding rows// are copied over the buffer its embed reads, measured: window 2's residual stream was window 1's stale tail).@@ -493,8 +542,11 @@ static void both_queues_barrier() {t_in_barrier = false;}+#endifError memset_device(void* p, int value, size_t bytes) {+#if defined(STRATA_DS4_RUNTIME)both_queues_barrier();+#endifStreamImpl* s = S(nullptr);const Error e = encode_fill(s, p, value, bytes);flush(s);@@ -502,11 +554,13 @@ Error memset_device(void* p, int value, size_t bytes) {return e;}Error memset_async(void* p, int value, size_t bytes, Stream st) {+#if defined(STRATA_DS4_RUNTIME)if (!S(st)->rec && ctx_queue_ops() && bytes > 0) {size_t o1 = 0;if (find(p, &o1) && ds4_ctx_fill(p, value, bytes)) return kSuccess;}if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();+#endifStreamImpl* s = S(st);if (s->rec) {Op o;@@ -518,7 +572,9 @@ Error memset_async(void* p, int value, size_t bytes, Stream st) {}Error memcpy_sync(void* dst, const void* src, size_t bytes, CopyKind) {+#if defined(STRATA_DS4_RUNTIME)both_queues_barrier();+#endifStreamImpl* s = S(nullptr);const Error e = encode_copy(s, dst, src, bytes);flush(s);@@ -537,11 +593,13 @@ Error memcpy_peer_async(void* dst, int dst_dev, const void* src, int src_dev, sireturn memcpy_async(dst, src, bytes, kDeviceToDevice, st);}Error memcpy_async(void* dst, const void* src, size_t bytes, CopyKind, Stream st) {+#if defined(STRATA_DS4_RUNTIME)if (!S(st)->rec && ctx_queue_ops() && bytes > 0) {size_t o1 = 0, o2 = 0;if (find(dst, &o1) && find(src, &o2) && ds4_ctx_copy(dst, src, bytes)) return kSuccess;}if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();+#endifStreamImpl* s = S(st);if (s->rec) {Op o;@@ -557,15 +615,19 @@ Error stream_destroy(Stream s) {if (!s) return kSuccess;flush(s);wait_value(s->tl, s->submitted);+#if defined(STRATA_DS4_RUNTIME){ std::lock_guard<std::mutex> lk(g_all_streams_mu);g_all_streams.erase(std::remove(g_all_streams.begin(), g_all_streams.end(), s), g_all_streams.end()); }+#endifs->tl->release();s->q->release();delete s;return kSuccess;}Error stream_synchronize(Stream st) {+#if defined(STRATA_DS4_RUNTIME)both_queues_barrier();+#endifStreamImpl* s = S(st);flush(s);return wait_value(s->tl, s->submitted);@@ -599,12 +661,14 @@ Error launch_host_func(Stream st, HostFn fn, void* arg) {Error event_create(Event* e, unsigned) { *e = new EventImpl; return kSuccess; }Error event_destroy(Event e) { delete e; return kSuccess; }Error event_record(Event e, Stream st) {+#if defined(STRATA_DS4_RUNTIME)if (ctx_queue_ops() && !S(st)->rec) { // the work the event follows is in the context's command buffer: end it, wait latere->token = ds4_ctx_mark();e->ev = nullptr;e->value = 0;return kSuccess;}+#endifStreamImpl* s = S(st);if (s->rec) {Op o;@@ -614,6 +678,7 @@ Error event_record(Event e, Stream st) {}return record_event(e, s);}+#if defined(STRATA_DS4_RUNTIME)Error event_query(Event e) {if (e->token) return kNotReady; // (a mark waits for its wait)return !e->ev || e->ev->signaledValue() >= e->value ? kSuccess : kNotReady;@@ -627,6 +692,10 @@ Error event_synchronize(Event e) {}return e->ev ? wait_value(e->ev, e->value) : kSuccess;}+#else+Error event_query(Event e) { return !e->ev || e->ev->signaledValue() >= e->value ? kSuccess : kNotReady; }+Error event_synchronize(Event e) { return e->ev ? wait_value(e->ev, e->value) : kSuccess; }+#endifError event_elapsed_ms(float* ms, Event, Event) { *ms = 0.0f; return fail(kErrorUnsupported); }Error begin_capture(Stream st) {@@ -737,6 +806,7 @@ Stats stats() { return rt().stats; }} // namespace metal} // namespace strata::gpu+#if defined(STRATA_DS4_RUNTIME)extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) {return strata::gpu::buffer_for_cpu_impl(cpu, bytes, off);}@@ -753,3 +823,5 @@ extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu) {if (!b) return 0;return ((MTL::Buffer*) b)->length() - off;}++#endif\ No newline at end of filediff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 9a29db4f1..92ec72e70 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -6,6 +6,7 @@#include "resource_path.h"#include <chrono>+#if defined(STRATA_DS4_RUNTIME)#include <execinfo.h>#include <algorithm>#include <atomic>@@ -16,6 +17,7 @@#include <mach/mach_time.h>#include <map>#include <thread>+#endif#include <cstdio>#include <cstdlib>#include <fstream>@@ -38,8 +40,11 @@ std::string read_file(const std::string& path) {return ss.str();}+#if defined(STRATA_DS4_RUNTIME)extern "C" void* ds4_gpu_residency_set() __attribute__((weak_import));+#endifstruct Context::Impl {+#if defined(STRATA_DS4_RUNTIME)id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate inid<MTLComputeCommandEncoder> enc_def = nil;int def_n = 0;@@ -49,6 +54,7 @@ struct Context::Impl {id<MTLCommandBuffer> inflight = nil; // the last committed-but-not-waited deferred buffer (the queue runs them in order)std::vector<void*> graveyard; // buffers freed while the deferred command buffer still references them (it holds none)std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoder+#endifid<MTLDevice> dev;id<MTLCommandQueue> q;std::vector<id<MTLLibrary>> libs;@@ -59,7 +65,11 @@ struct Context::Impl {static inline id<MTLBuffer> B(const Buffer& b) { return (__bridge id<MTLBuffer>)b.handle; }// the MTLBuffer and offset that bind byte `off` of b (a chunked buffer: the chunk the offset selects)static inline id<MTLBuffer> BO(const Buffer& b, size_t off, size_t& o) {+#if defined(STRATA_DS4_RUNTIME)if (!b.chunks) { o = off + b.bias; return B(b); }+#else+ if (!b.chunks) { o = off; return B(b); }+#endifconst size_t k = std::min(off / b.stride, b.chunks->size() - 1);o = off - k * b.stride;return (__bridge id<MTLBuffer>)(*b.chunks)[k];@@ -67,6 +77,7 @@ static inline id<MTLBuffer> BO(const Buffer& b, size_t off, size_t& o) {static inline id<MTLComputePipelineState> P(const Pipeline* p) { return (__bridge id<MTLComputePipelineState>)p->handle; }static std::string g_device_name = "unknown GPU";+#if defined(STRATA_DS4_RUNTIME)// ---- DS4_TIMELINE=1: every deferred dispatch in its own encoder with start / end GPU timestamps; the report at exit gives, per kernel,// its GPU time and the gap the GPU sat idle before it (host waits, event waits, launch). DS4_TL_FROM=N: only from dispatch N on.@@ -281,23 +292,28 @@ extern "C" void ds4_busy_end() {g_busy.running = false;}+#endifContext::Context() : impl_(new Impl) {impl_->dev = MTLCreateSystemDefaultDevice();+#if defined(STRATA_DS4_RUNTIME)if (std::getenv("DS4_TIMELINE") && !g_tl.reg) {g_tl.on = true;g_tl.reg = true;static id<MTLDevice> tl_dev = impl_->dev;std::atexit([] { tl_report(tl_dev); });}+#endifif (!impl_->dev) {std::fprintf(stderr, "mtl: no Metal device\n");std::exit(2);}impl_->q = [impl_->dev newCommandQueue];+#if defined(STRATA_DS4_RUNTIME)if (ds4_gpu_residency_set && !std::getenv("DS4_NO_RESIDENCY")) {id<MTLResidencySet> rs = (__bridge id<MTLResidencySet>)ds4_gpu_residency_set();if (rs) [impl_->q addResidencySet:rs];}+#endif// env, then <exe dir>/metal/ (a component package), then the compiled-in source-tree pathggml_common_path = strata_metal::resource("STRATA_GGML_COMMON", "ggml-common.h", STRATA_GGML_COMMON_DEFAULT);kernel_dir = strata_metal::resource("STRATA_METAL_KERNELS", "kernels", STRATA_METAL_KERNELS_DEFAULT);@@ -349,6 +365,7 @@ int Context::compile(const std::vector<std::string>& files, const std::string& preturn (int)impl_->libs.size() - 1;}+#if defined(STRATA_DS4_RUNTIME)// ---- the keep-alive spin (see mtl_rt.h)static std::atomic<int64_t> g_last_act_ns{0};@@ -433,7 +450,9 @@ uint64_t Context::zc_map(const void* p, size_t bytes) {}size_t Context::zc_count() const { std::lock_guard<std::mutex> lk(g_zc_mu); return g_zc.size(); }+#endifPipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {+#if defined(STRATA_DS4_RUNTIME)// (cached: the host helpers ask for the same pipeline a dispatch at a time; newComputePipelineState costs ~0.3 ms and// [MTLDevice name] reads the preferences - the prompt path's expert copies (3 a streamed expert) spent seconds there)static std::mutex cache_mu;@@ -455,6 +474,7 @@ Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {}Pipeline Context::pipeline_uncached(int lib, const std::string& fn, const Constants& c) {+#endifNSError* err = nil;id<MTLFunction> f;NSString* name = [NSString stringWithUTF8String:fn.c_str()];@@ -508,7 +528,11 @@ Pipeline Context::pipeline_uncached(int lib, const std::string& fn, const Constap.name += (i ? "," : "") + std::to_string(c.vals[i].index) + "=" + std::to_string((int)c.vals[i].v);p.name += "}";}+#if defined(STRATA_DS4_RUNTIME)if (g_device_name == "unknown GPU") g_device_name = [[impl_->dev name] UTF8String];+#else+ g_device_name = [[impl_->dev name] UTF8String];+#endifreturn p;}@@ -571,12 +595,15 @@ Buffer Context::wrap(void* p, size_t bytes, size_t max_span) {return r;}+#if defined(STRATA_DS4_RUNTIME)// The runtime's own MTLBuffer under a CPU address (strata::gpu's allocations), when [p, p + bytes) lies inside one. Binding it// - the same MTLBuffer, so the same GPU address, as every other kernel that touches this memory - instead of a fresh no-copy// buffer per call. (Each fresh wrapper is a second GPU mapping of the same pages: the quantiser read the activations through// one mapping while the norm that produced them had written through another, and saw the previous layer's.)extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) __attribute__((weak_import));+#endifBuffer Context::wrap(void* p, size_t bytes) {+#if defined(STRATA_DS4_RUNTIME)if (ds4_gpu_buffer_for_cpu && !std::getenv("DS4_WRAP_COPY")) {size_t off = 0;if (void* mb = ds4_gpu_buffer_for_cpu(p, bytes, &off)) {@@ -588,6 +615,7 @@ Buffer Context::wrap(void* p, size_t bytes) {return r;}}+#endifid<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];if (!b) {std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);@@ -602,50 +630,71 @@ Buffer Context::wrap(void* p, size_t bytes) {}void Context::free(Buffer& b) {+#if defined(STRATA_DS4_RUNTIME)// The deferred command buffer is made WITHOUT retained references: a buffer released here while a dispatch that names it// is still queued is read after it is gone (or after another alloc took its memory: the job arrays of iq_mv read the// NEXT call's jobs, measured as rows of zeros and different tokens every run). Hold it until the buffer has completed.std::lock_guard<std::recursive_mutex> lk_(impl_->mu);const bool hold = impl_->cb_def != nil || impl_->inflight != nil;+#endifif (b.chunks) {+#if defined(STRATA_DS4_RUNTIME)for (void* h : *b.chunks) {if (hold) impl_->graveyard.push_back(h);else { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }}+#else+ for (void* h : *b.chunks) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }+#endifb = Buffer{};return;}if (b.handle) {+#if defined(STRATA_DS4_RUNTIME)if (hold) impl_->graveyard.push_back(b.handle);else { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)b.handle; x = nil; }+#else+ id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)b.handle;+ x = nil;+#endif}b = Buffer{};}+#if defined(STRATA_DS4_RUNTIME)static std::mutex g_hist_mu;static std::unordered_map<void*, std::string> g_pipe_names;static std::unordered_map<std::string, uint64_t> g_kernel_counts;// DS4_ENC_TIMES: where a dispatch's encoding time goes - the pipeline state, the arguments, the dispatch callstatic std::atomic<uint64_t> g_t_pipe{0}, g_t_args{0}, g_t_disp{0};+#endifstatic void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {require_fit(d, "dispatch");+#if defined(STRATA_DS4_RUNTIME)static const bool enc_times = std::getenv("DS4_ENC_TIMES") != nullptr;const auto t_a = std::chrono::steady_clock::now();+#endif[enc setComputePipelineState:P(d.p)];+#if defined(STRATA_DS4_RUNTIME)const auto t_b = std::chrono::steady_clock::now();+#endiffor (size_t i = 0; i < d.args.size(); ++i) {const Arg& a = d.args[i];if (a.buf) {size_t o;id<MTLBuffer> mb = BO(*a.buf, a.off, o);+#if defined(STRATA_DS4_RUNTIME)static const bool chkbuf = std::getenv("DS4_CHECK_BUF") != nullptr;if (chkbuf && o >= [mb length]) std::fprintf(stderr, "mtl: %s arg %zu: offset %zu past the buffer (%zu bytes; Buffer.bytes %zu bias %zu off %zu)\n", d.p->name.c_str(), i, o, (size_t) [mb length], a.buf->bytes, a.buf->bias, (size_t) a.off);+#endif[enc setBuffer:mb offset:o atIndex:i];}else[enc setBytes:a.bytes.data() length:a.bytes.size() atIndex:i];}+#if defined(STRATA_DS4_RUNTIME)const auto t_c = std::chrono::steady_clock::now();+#endifif (d.tg_mem) [enc setThreadgroupMemoryLength:(d.tg_mem + 15) / 16 * 16 atIndex:0];if (d.indirect)[enc dispatchThreadgroupsWithIndirectBuffer:B(*d.indirect)@@ -654,6 +703,7 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {else[enc dispatchThreadgroups:MTLSizeMake(d.grid[0], d.grid[1], d.grid[2])threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];+#if defined(STRATA_DS4_RUNTIME)if (enc_times) {const auto t_d = std::chrono::steady_clock::now();g_t_pipe.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_b - t_a).count(), std::memory_order_relaxed);@@ -803,11 +853,15 @@ void Context::flush_runs() {}for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }impl_->graveyard.clear();+#endif}double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {+#if defined(STRATA_DS4_RUNTIME)std::lock_guard<std::recursive_mutex> lk_(impl_->mu);+#endif@autoreleasepool {+#if defined(STRATA_DS4_RUNTIME)if (ds.size() == 1 && reps == 1 && !barrier_between) {{ static const std::string skipk = std::getenv("DS4_SKIP_K") ? std::string(",") + std::getenv("DS4_SKIP_K") + "," : std::string(); // debugging: drop these kernels (timing only)if (!skipk.empty() && skipk.find("," + ds[0].p->name + ",") != std::string::npos) return 0.0; }@@ -860,13 +914,16 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,return 0.0;}flush_runs();+#endifid<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];id<MTLComputeCommandEncoder> enc =[cb computeCommandEncoderWithDispatchType:concurrent ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];for (int r = 0; r < reps; ++r)for (const auto& d : ds) {encode(enc, d);+#if defined(STRATA_DS4_RUNTIME)++g_disp_count;+#endifif (barrier_between) [enc memoryBarrierWithScope:MTLBarrierScopeBuffers];}[enc endEncoding];