Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal runtime: the DeepSeek queue/residency/barrier rules sit behind STRATA_DS4_RUNTIME; strata_ds4 compiles its own copy of mtl_rt.mm and gpu/runtime.cpp, the Qwen engine keeps the 0.2.7 code

Niko12216a51355051cf
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txt
index 46f048e90..b8ac13f57 100644
--- a/metal/CMakeLists.txt
+++ b/metal/CMakeLists.txt
@@ -143,7 +143,21 @@ list(APPEND STRATA_DS4_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/engine/exl3_legac
# The CPU lane for the experts the GPU does not hold (ggml-cpu's NEON row dots): without it every non-resident expert contributes
# zero and the residual stream stays the embedding (measured: layers 0-3 changed nothing, then random tokens).
set_source_files_properties(${DS4}/ds4_gg_cpu.cpp PROPERTIES COMPILE_DEFINITIONS "STRATA_DSV41_TQ_CPU=1")
-add_executable(strata_ds4 ${STRATA_DS4_SOURCES})
+# The DeepSeek engine's queue, residency and barrier rules (deferred command buffers, the second queue, the residency set shared with
+# strata::gpu, ...) live in mtl_rt.mm and gpu/runtime.cpp behind STRATA_DS4_RUNTIME. strata_ds4 compiles its own copy of these two
+# files with it; the Qwen engine (strata) and every other target keep the code of 0.2.7 (the macro is off there, so the files read
+# exactly as they did). The copies are linked as objects, so they win over the same symbols in the libraries.
+add_library(ds4_rt_mm OBJECT src/mtl_rt.mm)
+target_include_directories(ds4_rt_mm PRIVATE src ${STRATA_ROOT}/third_party/ggml ${STRATA_ROOT}/include)
+target_compile_options(ds4_rt_mm PRIVATE $<$<COMPILE_LANGUAGE:OBJCXX>:-fobjc-arc>)
+target_compile_definitions(ds4_rt_mm PRIVATE STRATA_DS4_RUNTIME=1 STRATA_GGML_COMMON_DEFAULT="${GGML_COMMON}"
+ STRATA_METAL_KERNELS_DEFAULT="${CMAKE_CURRENT_SOURCE_DIR}/kernels")
+add_library(ds4_rt_gpu OBJECT src/gpu/runtime.cpp)
+target_include_directories(ds4_rt_gpu BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat
+ ${CMAKE_CURRENT_SOURCE_DIR}/include ${STRATA_ROOT}/include)
+target_include_directories(ds4_rt_gpu SYSTEM PRIVATE ${METAL_CPP_DIR})
+target_compile_definitions(ds4_rt_gpu PRIVATE STRATA_DS4_RUNTIME=1 STRATA_GPU_METAL=1)
+add_executable(strata_ds4 ${STRATA_DS4_SOURCES} $<TARGET_OBJECTS:ds4_rt_mm> $<TARGET_OBJECTS:ds4_rt_gpu>)
target_include_directories(strata_ds4 PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat ${STRATA_ROOT}/include ${DS4} ${CMAKE_CURRENT_SOURCE_DIR}/src
${STRATA_ROOT}/third_party/ggml ${STRATA_GGML_DIR}/ggml/include ${STRATA_GGML_DIR}/ggml/src ${STRATA_GGML_DIR}/ggml/src/ggml-cpu)
diff --git a/metal/src/gpu/runtime.cpp b/metal/src/gpu/runtime.cpp
index cc8b054a7..f0547effa 100644
--- a/metal/src/gpu/runtime.cpp
+++ b/metal/src/gpu/runtime.cpp
@@ -24,13 +24,21 @@
#include <atomic>
#include <chrono>
#include <cstdio>
+#if defined(STRATA_DS4_RUNTIME)
#include <cstdlib>
+#endif
#include <cstring>
#include <iterator>
#include <map>
+#if defined(STRATA_DS4_RUNTIME)
#include <mutex> // recursive_mutex
+#else
+#include <mutex>
+#endif
#include <thread>
+#if defined(STRATA_DS4_RUNTIME)
#include <algorithm>
+#endif
#include <vector>
namespace strata::gpu {
@@ -68,7 +76,9 @@ struct Op {
struct EventImpl {
MTL::SharedEvent* ev = nullptr; // the recording stream's timeline
uint64_t value = 0; // 0: never recorded (complete, as a fresh CUDA event)
+#if defined(STRATA_DS4_RUNTIME)
void* token = nullptr; // a mark on the context's queue (ds4_ctx_mark), released by the wait
+#endif
};
struct GraphImpl {
@@ -121,12 +131,16 @@ Error fail(Error e) {
return e;
}
+#if defined(STRATA_DS4_RUNTIME)
static std::vector<StreamImpl*> g_all_streams; // every stream's queue holds pending copies until it is committed
static std::mutex g_all_streams_mu;
+#endif
StreamImpl* make_stream() {
auto* s = new StreamImpl;
+#if defined(STRATA_DS4_RUNTIME)
{ std::lock_guard<std::mutex> lk(g_all_streams_mu); g_all_streams.push_back(s); }
+#endif
s->q = rt().dev->newCommandQueue();
s->q->addResidencySet(rt().rs);
s->tl = rt().dev->newSharedEvent();
@@ -169,32 +183,47 @@ MTL::CommandBuffer* open_cb(StreamImpl* s) {
}
return s->cb;
}
+#if defined(STRATA_DS4_RUNTIME)
// StreamImpl's encoder/command-buffer state is touched from the loader, the prefetcher and the compute path.
// Without this lock two encoders can be open on one command buffer at once, which Metal asserts on.
static std::recursive_mutex g_stream_mu; // encode_copy/flush nest
+#endif
void end_encoder(StreamImpl* s) {
+#if defined(STRATA_DS4_RUNTIME)
if (s->enc) { if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] close compute enc %p\n", (void*) s->enc);
s->enc->endEncoding(); s->enc = nullptr; }
+#else
+ if (s->enc) { s->enc->endEncoding(); s->enc = nullptr; }
+#endif
}
MTL::ComputeCommandEncoder* encoder(StreamImpl* s) {
+#if defined(STRATA_DS4_RUNTIME)
std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
if (!s->enc) { s->enc = open_cb(s)->computeCommandEncoder();
if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] open compute enc %p (cb %p)\n", (void*) s->enc, (void*) s->cb); } // serial dispatch: each dispatch sees the last
+#else
+ if (!s->enc) s->enc = open_cb(s)->computeCommandEncoder(); // serial dispatch: each dispatch sees the last
+#endif
return s->enc;
}
void flush(StreamImpl* s) {
+#if defined(STRATA_DS4_RUNTIME)
std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
+#endif
if (!s->cb) return;
end_encoder(s);
+#if defined(STRATA_DS4_RUNTIME)
if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] signal %llu on cb %p (enc %p)\n",
(unsigned long long) (s->submitted + 1), (void*) s->cb, (void*) s->enc);
+#endif
s->cb->encodeSignalEvent(s->tl, ++s->submitted);
s->cb->commit();
s->cb->release();
s->cb = nullptr;
s->pending = 0;
}
+#if defined(STRATA_DS4_RUNTIME)
extern "C" void ds4_metal_flush();
extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) __attribute__((weak_import));
extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) __attribute__((weak_import));
@@ -205,6 +234,7 @@ static bool ctx_queue_ops() { // DS4_STREAM_COPIES=1: the old way (a copy on a
return on;
}
+#endif
Error wait_value(MTL::SharedEvent* ev, uint64_t v) {
while (ev->signaledValue() < v) std::this_thread::yield();
++rt().stats.host_syncs;
@@ -222,7 +252,9 @@ void check_purged(const Alloc* a, const char* what) {
}
Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {
+#if defined(STRATA_DS4_RUNTIME)
std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
+#endif
size_t od = 0, os = 0;
const Alloc* ad = find(dst, &od);
const Alloc* as = find(src, &os);
@@ -231,7 +263,9 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {
if (ad && as) { // buffer to buffer: a blit, in stream order
end_encoder(s);
MTL::BlitCommandEncoder* b = open_cb(s)->blitCommandEncoder();
+#if defined(STRATA_DS4_RUNTIME)
if (std::getenv("STRATA_GPU_TRACE")) std::fprintf(stderr, "[gpu] blit copy %zu\n", n);
+#endif
b->copyFromBuffer(as->buf, os, ad->buf, od, n);
b->endEncoding();
++rt().stats.blits;
@@ -248,7 +282,9 @@ Error encode_copy(StreamImpl* s, void* dst, const void* src, size_t n) {
}
Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {
+#if defined(STRATA_DS4_RUNTIME)
std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
+#endif
size_t off = 0;
const Alloc* a = find(p, &off);
if (!a) return fail(kErrorInvalidValue);
@@ -262,7 +298,9 @@ Error encode_fill(StreamImpl* s, void* p, int value, size_t n) {
}
Error encode_dispatch(StreamImpl* s, const Op& o) {
+#if defined(STRATA_DS4_RUNTIME)
std::lock_guard<std::recursive_mutex> lk(g_stream_mu);
+#endif
MTL::ComputeCommandEncoder* e = encoder(s);
e->setComputePipelineState(o.pipe);
if (o.big_args.empty()) e->setBytes(o.args, o.arg_bytes, 0);
@@ -308,8 +346,12 @@ void alloc_log(const char* what, size_t n) {
Error new_alloc(void** p, size_t bytes, bool host) {
Runtime& r = rt();
+#if defined(STRATA_DS4_RUNTIME)
const size_t pg = 16384, want = bytes ? bytes : 1;
const size_t n = std::getenv("DS4_NO_PAD") ? want : ((want + pg - 1) / pg) * pg;
+#else
+ const size_t n = bytes ? bytes : 1;
+#endif
MTL::Buffer* b = r.dev->newBuffer(n, MTL::ResourceStorageModeShared | MTL::ResourceHazardTrackingModeUntracked);
if (!b) return fail(kErrorMemoryAllocation);
Alloc a{b, b->gpuAddress(), (uint8_t*) b->contents(), n, host};
@@ -331,7 +373,9 @@ Error free_alloc(void* p) {
size_t off = 0;
const Alloc* a = find(p, &off);
if (!a || off != 0) return fail(kErrorInvalidValue);
+#if defined(STRATA_DS4_RUNTIME)
if (std::getenv("DS4_TRACE_FREE")) std::fprintf(stderr, "gpu free %zu bytes (host=%d) at %p\n", a->bytes, (int) a->host, p);
+#endif
device_synchronize(); // as cudaFree: the GPU is done with it
std::lock_guard<std::mutex> lk(r.mu);
MTL::Buffer* b = a->buf;
@@ -347,6 +391,7 @@ Error free_alloc(void* p) {
} // namespace
// ---------------------------------------------------------------------------------------------------- runtime.hpp
+#if defined(STRATA_DS4_RUNTIME)
void* buffer_for_cpu_impl(const void* cpu, size_t bytes, size_t* off) {
Runtime& r = rt();
std::lock_guard<std::mutex> lk(r.mu);
@@ -361,6 +406,7 @@ void* buffer_for_cpu_impl(const void* cpu, size_t bytes, size_t* off) {
MTL::ResidencySet* residency_set_impl() { return rt().rs; }
+#endif
void* host_view(const void* p) {
size_t off = 0;
const Alloc* a = find(p, &off);
@@ -405,7 +451,9 @@ Error get_device(int* dev) { *dev = 0; return kSuccess; }
Error set_device(int dev) { return dev == 0 ? kSuccess : fail(kErrorInvalidValue); }
Error device_count(int* n) { *n = rt().dev ? 1 : 0; return kSuccess; }
Error device_synchronize() {
+#if defined(STRATA_DS4_RUNTIME)
ds4_metal_flush();
+#endif
Runtime& r = rt();
StreamImpl* s = S(nullptr);
flush(s);
@@ -475,6 +523,7 @@ Error host_device_pointer_raw(void** dev, void* host) {
*dev = (void*) (uintptr_t) (a->gpu + off);
return kSuccess;
}
+#if defined(STRATA_DS4_RUNTIME)
// A synchronising call is a barrier for BOTH queues: the dispatches deferred in mtl_rt's command buffer have to run before
// the host reads or overwrites what they touch (a window's last layers still pending while the next window's embedding rows
// are copied over the buffer its embed reads, measured: window 2's residual stream was window 1's stale tail).
@@ -493,8 +542,11 @@ static void both_queues_barrier() {
t_in_barrier = false;
}
+#endif
Error memset_device(void* p, int value, size_t bytes) {
+#if defined(STRATA_DS4_RUNTIME)
both_queues_barrier();
+#endif
StreamImpl* s = S(nullptr);
const Error e = encode_fill(s, p, value, bytes);
flush(s);
@@ -502,11 +554,13 @@ Error memset_device(void* p, int value, size_t bytes) {
return e;
}
Error memset_async(void* p, int value, size_t bytes, Stream st) {
+#if defined(STRATA_DS4_RUNTIME)
if (!S(st)->rec && ctx_queue_ops() && bytes > 0) {
size_t o1 = 0;
if (find(p, &o1) && ds4_ctx_fill(p, value, bytes)) return kSuccess;
}
if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();
+#endif
StreamImpl* s = S(st);
if (s->rec) {
Op o;
@@ -518,7 +572,9 @@ Error memset_async(void* p, int value, size_t bytes, Stream st) {
}
Error memcpy_sync(void* dst, const void* src, size_t bytes, CopyKind) {
+#if defined(STRATA_DS4_RUNTIME)
both_queues_barrier();
+#endif
StreamImpl* s = S(nullptr);
const Error e = encode_copy(s, dst, src, bytes);
flush(s);
@@ -537,11 +593,13 @@ Error memcpy_peer_async(void* dst, int dst_dev, const void* src, int src_dev, si
return memcpy_async(dst, src, bytes, kDeviceToDevice, st);
}
Error memcpy_async(void* dst, const void* src, size_t bytes, CopyKind, Stream st) {
+#if defined(STRATA_DS4_RUNTIME)
if (!S(st)->rec && ctx_queue_ops() && bytes > 0) {
size_t o1 = 0, o2 = 0;
if (find(dst, &o1) && find(src, &o2) && ds4_ctx_copy(dst, src, bytes)) return kSuccess;
}
if (!S(st)->rec && !std::getenv("DS4_NO_QBARRIER")) both_queues_barrier();
+#endif
StreamImpl* s = S(st);
if (s->rec) {
Op o;
@@ -557,15 +615,19 @@ Error stream_destroy(Stream s) {
if (!s) return kSuccess;
flush(s);
wait_value(s->tl, s->submitted);
+#if defined(STRATA_DS4_RUNTIME)
{ std::lock_guard<std::mutex> lk(g_all_streams_mu);
g_all_streams.erase(std::remove(g_all_streams.begin(), g_all_streams.end(), s), g_all_streams.end()); }
+#endif
s->tl->release();
s->q->release();
delete s;
return kSuccess;
}
Error stream_synchronize(Stream st) {
+#if defined(STRATA_DS4_RUNTIME)
both_queues_barrier();
+#endif
StreamImpl* s = S(st);
flush(s);
return wait_value(s->tl, s->submitted);
@@ -599,12 +661,14 @@ Error launch_host_func(Stream st, HostFn fn, void* arg) {
Error event_create(Event* e, unsigned) { *e = new EventImpl; return kSuccess; }
Error event_destroy(Event e) { delete e; return kSuccess; }
Error event_record(Event e, Stream st) {
+#if defined(STRATA_DS4_RUNTIME)
if (ctx_queue_ops() && !S(st)->rec) { // the work the event follows is in the context's command buffer: end it, wait later
e->token = ds4_ctx_mark();
e->ev = nullptr;
e->value = 0;
return kSuccess;
}
+#endif
StreamImpl* s = S(st);
if (s->rec) {
Op o;
@@ -614,6 +678,7 @@ Error event_record(Event e, Stream st) {
}
return record_event(e, s);
}
+#if defined(STRATA_DS4_RUNTIME)
Error event_query(Event e) {
if (e->token) return kNotReady; // (a mark waits for its wait)
return !e->ev || e->ev->signaledValue() >= e->value ? kSuccess : kNotReady;
@@ -627,6 +692,10 @@ Error event_synchronize(Event e) {
}
return e->ev ? wait_value(e->ev, e->value) : kSuccess;
}
+#else
+Error event_query(Event e) { return !e->ev || e->ev->signaledValue() >= e->value ? kSuccess : kNotReady; }
+Error event_synchronize(Event e) { return e->ev ? wait_value(e->ev, e->value) : kSuccess; }
+#endif
Error event_elapsed_ms(float* ms, Event, Event) { *ms = 0.0f; return fail(kErrorUnsupported); }
Error begin_capture(Stream st) {
@@ -737,6 +806,7 @@ Stats stats() { return rt().stats; }
} // namespace metal
} // namespace strata::gpu
+#if defined(STRATA_DS4_RUNTIME)
extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) {
return strata::gpu::buffer_for_cpu_impl(cpu, bytes, off);
}
@@ -753,3 +823,5 @@ extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu) {
if (!b) return 0;
return ((MTL::Buffer*) b)->length() - off;
}
+
+#endif
\ No newline at end of file
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index 9a29db4f1..92ec72e70 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -6,6 +6,7 @@
#include "resource_path.h"
#include <chrono>
+#if defined(STRATA_DS4_RUNTIME)
#include <execinfo.h>
#include <algorithm>
#include <atomic>
@@ -16,6 +17,7 @@
#include <mach/mach_time.h>
#include <map>
#include <thread>
+#endif
#include <cstdio>
#include <cstdlib>
#include <fstream>
@@ -38,8 +40,11 @@ std::string read_file(const std::string& path) {
return ss.str();
}
+#if defined(STRATA_DS4_RUNTIME)
extern "C" void* ds4_gpu_residency_set() __attribute__((weak_import));
+#endif
struct Context::Impl {
+#if defined(STRATA_DS4_RUNTIME)
id<MTLCommandBuffer> cb_def = nil; // the open command buffer single dispatches accumulate in
id<MTLComputeCommandEncoder> enc_def = nil;
int def_n = 0;
@@ -49,6 +54,7 @@ struct Context::Impl {
id<MTLCommandBuffer> inflight = nil; // the last committed-but-not-waited deferred buffer (the queue runs them in order)
std::vector<void*> graveyard; // buffers freed while the deferred command buffer still references them (it holds none)
std::recursive_mutex mu; // helper threads (the stager, the expert cache) flush too: one writer on the open encoder
+#endif
id<MTLDevice> dev;
id<MTLCommandQueue> q;
std::vector<id<MTLLibrary>> libs;
@@ -59,7 +65,11 @@ struct Context::Impl {
static inline id<MTLBuffer> B(const Buffer& b) { return (__bridge id<MTLBuffer>)b.handle; }
// the MTLBuffer and offset that bind byte `off` of b (a chunked buffer: the chunk the offset selects)
static inline id<MTLBuffer> BO(const Buffer& b, size_t off, size_t& o) {
+#if defined(STRATA_DS4_RUNTIME)
if (!b.chunks) { o = off + b.bias; return B(b); }
+#else
+ if (!b.chunks) { o = off; return B(b); }
+#endif
const size_t k = std::min(off / b.stride, b.chunks->size() - 1);
o = off - k * b.stride;
return (__bridge id<MTLBuffer>)(*b.chunks)[k];
@@ -67,6 +77,7 @@ static inline id<MTLBuffer> BO(const Buffer& b, size_t off, size_t& o) {
static inline id<MTLComputePipelineState> P(const Pipeline* p) { return (__bridge id<MTLComputePipelineState>)p->handle; }
static std::string g_device_name = "unknown GPU";
+#if defined(STRATA_DS4_RUNTIME)
// ---- DS4_TIMELINE=1: every deferred dispatch in its own encoder with start / end GPU timestamps; the report at exit gives, per kernel,
// its GPU time and the gap the GPU sat idle before it (host waits, event waits, launch). DS4_TL_FROM=N: only from dispatch N on.
@@ -281,23 +292,28 @@ extern "C" void ds4_busy_end() {
g_busy.running = false;
}
+#endif
Context::Context() : impl_(new Impl) {
impl_->dev = MTLCreateSystemDefaultDevice();
+#if defined(STRATA_DS4_RUNTIME)
if (std::getenv("DS4_TIMELINE") && !g_tl.reg) {
g_tl.on = true;
g_tl.reg = true;
static id<MTLDevice> tl_dev = impl_->dev;
std::atexit([] { tl_report(tl_dev); });
}
+#endif
if (!impl_->dev) {
std::fprintf(stderr, "mtl: no Metal device\n");
std::exit(2);
}
impl_->q = [impl_->dev newCommandQueue];
+#if defined(STRATA_DS4_RUNTIME)
if (ds4_gpu_residency_set && !std::getenv("DS4_NO_RESIDENCY")) {
id<MTLResidencySet> rs = (__bridge id<MTLResidencySet>)ds4_gpu_residency_set();
if (rs) [impl_->q addResidencySet:rs];
}
+#endif
// env, then <exe dir>/metal/ (a component package), then the compiled-in source-tree path
ggml_common_path = strata_metal::resource("STRATA_GGML_COMMON", "ggml-common.h", STRATA_GGML_COMMON_DEFAULT);
kernel_dir = strata_metal::resource("STRATA_METAL_KERNELS", "kernels", STRATA_METAL_KERNELS_DEFAULT);
@@ -349,6 +365,7 @@ int Context::compile(const std::vector<std::string>& files, const std::string& p
return (int)impl_->libs.size() - 1;
}
+#if defined(STRATA_DS4_RUNTIME)
// ---- the keep-alive spin (see mtl_rt.h)
static std::atomic<int64_t> g_last_act_ns{0};
@@ -433,7 +450,9 @@ uint64_t Context::zc_map(const void* p, size_t bytes) {
}
size_t Context::zc_count() const { std::lock_guard<std::mutex> lk(g_zc_mu); return g_zc.size(); }
+#endif
Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {
+#if defined(STRATA_DS4_RUNTIME)
// (cached: the host helpers ask for the same pipeline a dispatch at a time; newComputePipelineState costs ~0.3 ms and
// [MTLDevice name] reads the preferences - the prompt path's expert copies (3 a streamed expert) spent seconds there)
static std::mutex cache_mu;
@@ -455,6 +474,7 @@ Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {
}
Pipeline Context::pipeline_uncached(int lib, const std::string& fn, const Constants& c) {
+#endif
NSError* err = nil;
id<MTLFunction> f;
NSString* name = [NSString stringWithUTF8String:fn.c_str()];
@@ -508,7 +528,11 @@ Pipeline Context::pipeline_uncached(int lib, const std::string& fn, const Consta
p.name += (i ? "," : "") + std::to_string(c.vals[i].index) + "=" + std::to_string((int)c.vals[i].v);
p.name += "}";
}
+#if defined(STRATA_DS4_RUNTIME)
if (g_device_name == "unknown GPU") g_device_name = [[impl_->dev name] UTF8String];
+#else
+ g_device_name = [[impl_->dev name] UTF8String];
+#endif
return p;
}
@@ -571,12 +595,15 @@ Buffer Context::wrap(void* p, size_t bytes, size_t max_span) {
return r;
}
+#if defined(STRATA_DS4_RUNTIME)
// The runtime's own MTLBuffer under a CPU address (strata::gpu's allocations), when [p, p + bytes) lies inside one. Binding it
// - the same MTLBuffer, so the same GPU address, as every other kernel that touches this memory - instead of a fresh no-copy
// buffer per call. (Each fresh wrapper is a second GPU mapping of the same pages: the quantiser read the activations through
// one mapping while the norm that produced them had written through another, and saw the previous layer's.)
extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) __attribute__((weak_import));
+#endif
Buffer Context::wrap(void* p, size_t bytes) {
+#if defined(STRATA_DS4_RUNTIME)
if (ds4_gpu_buffer_for_cpu && !std::getenv("DS4_WRAP_COPY")) {
size_t off = 0;
if (void* mb = ds4_gpu_buffer_for_cpu(p, bytes, &off)) {
@@ -588,6 +615,7 @@ Buffer Context::wrap(void* p, size_t bytes) {
return r;
}
}
+#endif
id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];
if (!b) {
std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);
@@ -602,50 +630,71 @@ Buffer Context::wrap(void* p, size_t bytes) {
}
void Context::free(Buffer& b) {
+#if defined(STRATA_DS4_RUNTIME)
// The deferred command buffer is made WITHOUT retained references: a buffer released here while a dispatch that names it
// is still queued is read after it is gone (or after another alloc took its memory: the job arrays of iq_mv read the
// NEXT call's jobs, measured as rows of zeros and different tokens every run). Hold it until the buffer has completed.
std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
const bool hold = impl_->cb_def != nil || impl_->inflight != nil;
+#endif
if (b.chunks) {
+#if defined(STRATA_DS4_RUNTIME)
for (void* h : *b.chunks) {
if (hold) impl_->graveyard.push_back(h);
else { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
}
+#else
+ for (void* h : *b.chunks) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
+#endif
b = Buffer{};
return;
}
if (b.handle) {
+#if defined(STRATA_DS4_RUNTIME)
if (hold) impl_->graveyard.push_back(b.handle);
else { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)b.handle; x = nil; }
+#else
+ id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)b.handle;
+ x = nil;
+#endif
}
b = Buffer{};
}
+#if defined(STRATA_DS4_RUNTIME)
static std::mutex g_hist_mu;
static std::unordered_map<void*, std::string> g_pipe_names;
static std::unordered_map<std::string, uint64_t> g_kernel_counts;
// DS4_ENC_TIMES: where a dispatch's encoding time goes - the pipeline state, the arguments, the dispatch call
static std::atomic<uint64_t> g_t_pipe{0}, g_t_args{0}, g_t_disp{0};
+#endif
static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
require_fit(d, "dispatch");
+#if defined(STRATA_DS4_RUNTIME)
static const bool enc_times = std::getenv("DS4_ENC_TIMES") != nullptr;
const auto t_a = std::chrono::steady_clock::now();
+#endif
[enc setComputePipelineState:P(d.p)];
+#if defined(STRATA_DS4_RUNTIME)
const auto t_b = std::chrono::steady_clock::now();
+#endif
for (size_t i = 0; i < d.args.size(); ++i) {
const Arg& a = d.args[i];
if (a.buf) {
size_t o;
id<MTLBuffer> mb = BO(*a.buf, a.off, o);
+#if defined(STRATA_DS4_RUNTIME)
static const bool chkbuf = std::getenv("DS4_CHECK_BUF") != nullptr;
if (chkbuf && o >= [mb length]) std::fprintf(stderr, "mtl: %s arg %zu: offset %zu past the buffer (%zu bytes; Buffer.bytes %zu bias %zu off %zu)\n", d.p->name.c_str(), i, o, (size_t) [mb length], a.buf->bytes, a.buf->bias, (size_t) a.off);
+#endif
[enc setBuffer:mb offset:o atIndex:i];
}
else
[enc setBytes:a.bytes.data() length:a.bytes.size() atIndex:i];
}
+#if defined(STRATA_DS4_RUNTIME)
const auto t_c = std::chrono::steady_clock::now();
+#endif
if (d.tg_mem) [enc setThreadgroupMemoryLength:(d.tg_mem + 15) / 16 * 16 atIndex:0];
if (d.indirect)
[enc dispatchThreadgroupsWithIndirectBuffer:B(*d.indirect)
@@ -654,6 +703,7 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
else
[enc dispatchThreadgroups:MTLSizeMake(d.grid[0], d.grid[1], d.grid[2])
threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];
+#if defined(STRATA_DS4_RUNTIME)
if (enc_times) {
const auto t_d = std::chrono::steady_clock::now();
g_t_pipe.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_b - t_a).count(), std::memory_order_relaxed);
@@ -803,11 +853,15 @@ void Context::flush_runs() {
}
for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
impl_->graveyard.clear();
+#endif
}
double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {
+#if defined(STRATA_DS4_RUNTIME)
std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
+#endif
@autoreleasepool {
+#if defined(STRATA_DS4_RUNTIME)
if (ds.size() == 1 && reps == 1 && !barrier_between) {
{ static const std::string skipk = std::getenv("DS4_SKIP_K") ? std::string(",") + std::getenv("DS4_SKIP_K") + "," : std::string(); // debugging: drop these kernels (timing only)
if (!skipk.empty() && skipk.find("," + ds[0].p->name + ",") != std::string::npos) return 0.0; }
@@ -860,13 +914,16 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,
return 0.0;
}
flush_runs();
+#endif
id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];
id<MTLComputeCommandEncoder> enc =
[cb computeCommandEncoderWithDispatchType:concurrent ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];
for (int r = 0; r < reps; ++r)
for (const auto& d : ds) {
encode(enc, d);
+#if defined(STRATA_DS4_RUNTIME)
++g_disp_count;
+#endif
if (barrier_between) [enc memoryBarrierWithScope:MTLBarrierScopeBuffers];
}
[enc endEncoding];