Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
55.2 KB
Raw Blame
// metal/src/mtl_rt.mm - see mtl_rt.h.  Built with ARC (-fobjc-arc).
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>

#include "mtl_rt.h"
#include "resource_path.h"

#include <chrono>
#if defined(STRATA_DS4_RUNTIME)
#include <execinfo.h>
#include <algorithm>
#include <atomic>
#include <mutex>
#include <unordered_map>
#include <string>
#include <deque>
#include <mach/mach_time.h>
#include <map>
#include <thread>
#endif
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <sstream>

namespace mtl {

double now_s() {
    return std::chrono::duration<double>(std::chrono::steady_clock::now().time_since_epoch()).count();
}

std::string read_file(const std::string& path) {
    std::ifstream f(path, std::ios::binary);
    if (!f) {
        std::fprintf(stderr, "mtl: cannot read %s\n", path.c_str());
        std::exit(2);
    }
    std::stringstream ss;
    ss << f.rdbuf();
    return ss.str();
}

#if defined(STRATA_DS4_RUNTIME)
extern "C" void* ds4_gpu_residency_set() __attribute__((weak_import));
#endif
struct Context::Impl {
#if defined(STRATA_DS4_RUNTIME)
    id<MTLCommandBuffer> cb_def = nil;   // the open command buffer single dispatches accumulate in
    id<MTLComputeCommandEncoder> enc_def = nil;
    int def_n = 0;
    std::string def_names, inflight_names;   // (DS4_ERRINFO) the kernels of the open / the committed command buffer
    id<MTLSharedEvent> qev = nil;          // consecutive deferred buffers are chained through it (untracked buffers: the queue alone does not order them)
    uint64_t qev_v = 0;
    id<MTLCommandBuffer> inflight = nil;   // the last committed-but-not-waited deferred buffer (the queue runs them in order)
    std::vector<void*> graveyard;   // buffers freed while the deferred command buffer still references them (it holds none)
    std::recursive_mutex mu;   // helper threads (the stager, the expert cache) flush too: one writer on the open encoder
#endif
    id<MTLDevice> dev;
    id<MTLCommandQueue> q;
    std::vector<id<MTLLibrary>> libs;
    id<MTLSharedEvent> ev = nil;   // run_graph_seg's handshake timeline
    uint64_t ev_v = 0;
};

static inline id<MTLBuffer> B(const Buffer& b) { return (__bridge id<MTLBuffer>)b.handle; }
// the MTLBuffer and offset that bind byte `off` of b (a chunked buffer: the chunk the offset selects)
static inline id<MTLBuffer> BO(const Buffer& b, size_t off, size_t& o) {
#if defined(STRATA_DS4_RUNTIME)
    if (!b.chunks) { o = off + b.bias; return B(b); }
#else
    if (!b.chunks) { o = off; return B(b); }
#endif
    const size_t k = std::min(off / b.stride, b.chunks->size() - 1);
    o = off - k * b.stride;
    return (__bridge id<MTLBuffer>)(*b.chunks)[k];
}
static inline id<MTLComputePipelineState> P(const Pipeline* p) { return (__bridge id<MTLComputePipelineState>)p->handle; }
static std::string g_device_name = "unknown GPU";

#if defined(STRATA_DS4_RUNTIME)

// ---- DS4_TIMELINE=1: every deferred dispatch in its own encoder with start / end GPU timestamps; the report at exit gives, per kernel,
// its GPU time and the gap the GPU sat idle before it (host waits, event waits, launch).  DS4_TL_FROM=N: only from dispatch N on.
static std::atomic<uint64_t> g_disp_count{0};   // dispatches encoded
namespace {
struct Timeline {
    bool on = false;
    id<MTLCounterSet> tset = nil;
    std::vector<id<MTLCounterSampleBuffer>> bufs;   // a sample buffer holds only ~1500 encoders: they rotate
    size_t per = 0, cap = 0, n = 0;        // encoders per buffer / in all (the buffers' limit) / recorded
    long after = 0;                        // DS4_TL_AFTER: record from the Nth dispatch on
    std::vector<std::string> names;        // per encoder
    uint64_t cpu0 = 0, gpu0 = 0;
    bool reg = false;
    bool gate = true;                      // ds4_timeline_gate: record only while open
    bool att = false;                      // the encoder just made carries a sample slot
} g_tl;

id<MTLComputeCommandEncoder> tl_make_encoder(id<MTLDevice> dev, id<MTLCommandBuffer> cb, MTLDispatchType dt) {
    g_tl.att = false;
    if (!g_tl.on) return [cb computeCommandEncoderWithDispatchType:dt];
    if (g_tl.tset == nil) {
        for (id<MTLCounterSet> cs in dev.counterSets)
            if ([cs.name isEqualToString:MTLCommonCounterSetTimestamp]) g_tl.tset = cs;
        if (g_tl.tset == nil) {
            std::fprintf(stderr, "mtl: DS4_TIMELINE: no timestamp counter set\n");
            g_tl.on = false;
            return [cb computeCommandEncoderWithDispatchType:dt];
        }
        for (size_t c = 4096; c >= 64 && g_tl.per == 0; c /= 2) {
            MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];
            d.counterSet = g_tl.tset;
            d.storageMode = MTLStorageModeShared;
            d.sampleCount = c * 2;
            NSError* err = nil;
            id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];
            if (b != nil) {
                g_tl.per = c;
                g_tl.bufs.push_back(b);
            }
        }
        if (g_tl.per == 0) {
            std::fprintf(stderr, "mtl: DS4_TIMELINE: no counter sample buffer\n");
            g_tl.on = false;
            return [cb computeCommandEncoderWithDispatchType:dt];
        }
        g_tl.cap = g_tl.per * 120;
        MTLTimestamp c0 = 0, g0 = 0;
        [dev sampleTimestamps:&c0 gpuTimestamp:&g0];
        g_tl.cpu0 = c0;
        g_tl.gpu0 = g0;
        g_tl.names.assign(g_tl.cap, std::string());
        g_tl.after = std::getenv("DS4_TL_AFTER") ? std::atol(std::getenv("DS4_TL_AFTER")) : 0;
        if (std::getenv("DS4_TL_DRAFT")) g_tl.gate = false;
    }
    if (!g_tl.gate || g_tl.n >= g_tl.cap || (long) g_disp_count.load() < g_tl.after) return [cb computeCommandEncoderWithDispatchType:dt];
    const size_t bi = g_tl.n / g_tl.per, si = g_tl.n % g_tl.per;
    if (bi >= g_tl.bufs.size()) {
        MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];
        d.counterSet = g_tl.tset;
        d.storageMode = MTLStorageModeShared;
        d.sampleCount = g_tl.per * 2;
        NSError* err = nil;
        id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];
        if (b == nil) {
            g_tl.cap = g_tl.n;
            return [cb computeCommandEncoderWithDispatchType:dt];
        }
        g_tl.bufs.push_back(b);
    }
    MTLComputePassDescriptor* pd = [MTLComputePassDescriptor computePassDescriptor];
    pd.dispatchType = dt;
    pd.sampleBufferAttachments[0].sampleBuffer = g_tl.bufs[bi];
    pd.sampleBufferAttachments[0].startOfEncoderSampleIndex = 2 * si;
    pd.sampleBufferAttachments[0].endOfEncoderSampleIndex = 2 * si + 1;
    g_tl.att = true;
    return [cb computeCommandEncoderWithDescriptor:pd];
}

void tl_report(id<MTLDevice> dev) {
    if (!g_tl.on && g_tl.bufs.empty()) return;
    const size_t n = std::min(g_tl.n, g_tl.cap);
    if (n < 2) return;
    std::vector<MTLCounterResultTimestamp> tsv(2 * n);
    for (size_t b = 0; b * g_tl.per < n; ++b) {
        const size_t cnt = std::min(g_tl.per, n - b * g_tl.per);
        NSData* data = [g_tl.bufs[b] resolveCounterRange:NSMakeRange(0, 2 * cnt)];
        if (data == nil) { std::fprintf(stderr, "mtl: DS4_TIMELINE: resolve failed\n"); return; }
        std::memcpy(&tsv[2 * b * g_tl.per], [data bytes], 2 * cnt * sizeof(MTLCounterResultTimestamp));
    }
    const MTLCounterResultTimestamp* ts = tsv.data();
    MTLTimestamp c1 = 0, g1 = 0;
    [dev sampleTimestamps:&c1 gpuTimestamp:&g1];
    mach_timebase_info_data_t tb;
    mach_timebase_info(&tb);
    const double cpu_ns_per_tick = (double) tb.numer / (double) tb.denom;
    (void) cpu_ns_per_tick;   // (sampleTimestamps' CPU value is in ns already)
    const double ns_per_gpu = (g1 > g_tl.gpu0) ? (double) (c1 - g_tl.cpu0) / (double) (g1 - g_tl.gpu0) : 1.0;
    std::fprintf(stderr, "timeline: cap %zu n %zu scale %.6f ns/tick first ts %llu %llu last %llu\n", g_tl.cap, g_tl.n, ns_per_gpu, (unsigned long long) ts[0].timestamp, (unsigned long long) ts[1].timestamp, (unsigned long long) ts[2 * n - 1].timestamp);
    size_t from = std::getenv("DS4_TL_FROM") ? (size_t) std::atoll(std::getenv("DS4_TL_FROM")) : 0;
    struct Acc { double busy = 0, gap = 0, gap_big = 0; size_t n = 0, n_big = 0; };
    std::map<std::string, Acc> acc;
    double t_first = 0, t_last = 0, total_busy = 0, total_gap = 0, total_big = 0;
    double prev_end = -1, ovl_us = 0;
    size_t used = 0, n_ovl = 0;
    for (size_t k = from; k < n; ++k) {
        if (g_tl.names[k].empty()) continue;
        const double st = (double) ts[2 * k].timestamp * ns_per_gpu * 1e-3, en = (double) ts[2 * k + 1].timestamp * ns_per_gpu * 1e-3;   // us
        if (!(en >= st)) continue;
        if (prev_end < 0) t_first = st;
        Acc& a = acc[g_tl.names[k]];
        a.busy += en - st;
        ++a.n;
        total_busy += en - st;
        if (prev_end >= 0 && st < prev_end) { ++n_ovl; ovl_us += prev_end - st; }
        if (prev_end >= 0 && st >= prev_end) {
            const double gp = st - prev_end;
            a.gap += gp;
            total_gap += gp;
            if (gp > 300.0) { a.gap_big += gp; ++a.n_big; total_big += gp; }
        }
        prev_end = en;
        t_last = en;
        ++used;
    }
    std::vector<std::pair<double, std::string>> v;
    for (auto& kv : acc) v.emplace_back(kv.second.busy + kv.second.gap, kv.first);
    std::sort(v.begin(), v.end(), [](auto& x, auto& y) { return x.first > y.first; });
    std::fprintf(stderr, "timeline: %zu dispatches, wall %.1f ms, GPU busy %.1f ms, idle gaps %.1f ms (of it > 300 us: %.1f ms)\n", used,
                 (t_last - t_first) / 1000.0, total_busy / 1000.0, total_gap / 1000.0, total_big / 1000.0);
    std::fprintf(stderr, "timeline: %zu dispatches started before the previous one ended (%.1f ms overlapped)\n", n_ovl, ovl_us / 1000.0);
    std::fprintf(stderr, "timeline: kernel                          n   busy ms  us/disp   gap ms  gap/disp  big-gap ms (n)\n");
    for (size_t i = 0; i < v.size() && i < 40; ++i) {
        const Acc& a = acc[v[i].second];
        std::fprintf(stderr, "timeline: %-28s %6zu %9.1f %8.1f %8.1f %8.1f %10.1f (%zu)\n", v[i].second.c_str(), a.n, a.busy / 1000.0, a.busy / (double) a.n,
                     a.gap / 1000.0, a.gap / (double) a.n, a.gap_big / 1000.0, a.n_big);
    }
}
}  // namespace

extern "C" void ds4_timeline_gate(int open) { g_tl.gate = open != 0; }


// ---- ds4_busy_begin / ds4_busy_end: a one-threadgroup kernel on a second queue that spins on a shared flag while the host works
// (the CPU lane), so the GPU is not idle between two bursts (idle -> the next kernels run 2-3x slower); the host sets the flag the
// moment it is done and the kernel leaves within microseconds.  DS4_BUSY=0 turns it off.
namespace {
struct Busy {
    bool init = false, ok = false;
    id<MTLCommandQueue> q = nil;
    id<MTLComputePipelineState> ps = nil;
    id<MTLBuffer> flag = nil;
    id<MTLCommandBuffer> last = nil;
    bool running = false;
};
Busy g_busy;
}  // namespace

extern "C" void ds4_busy_begin() {
    static const bool off = std::getenv("DS4_BUSY") && std::string(std::getenv("DS4_BUSY")) == "0";
    if (off) return;
    @autoreleasepool {
        if (!g_busy.init) {
            g_busy.init = true;
            id<MTLDevice> dev = MTLCreateSystemDefaultDevice();
            static const char* kSrc = R"MSL(
#include <metal_stdlib>
using namespace metal;
kernel void busy_spin(device volatile atomic_uint* flag [[buffer(0)]], constant uint& cap [[buffer(1)]], uint t [[thread_position_in_grid]]) {
    uint x = t;
    float f = (float) t * 1e-9f;
    for (uint i = 0; i < cap; ++i) {
        for (uint j = 0; j < 64; ++j) x = x * 1664525u + 1013904223u;
        for (uint j = 0; j < 16; ++j) f = fma(f, 1.0000001f, 1e-7f);   // (a real FP chain: keeps the ALU clock up)
        x ^= as_type<uint>(f);
        if (atomic_load_explicit(flag, memory_order_relaxed) != 0u) break;
    }
    if (x == 0xdeadbeefu) atomic_store_explicit(flag, 2u, memory_order_relaxed);
}
)MSL";
            NSError* err = nil;
            id<MTLLibrary> lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kSrc] options:nil error:&err];
            if (lib) g_busy.ps = [dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"busy_spin"] error:&err];
            if (g_busy.ps) {
                g_busy.q = [dev newCommandQueue];
                g_busy.flag = [dev newBufferWithLength:64 options:MTLResourceStorageModeShared];
                g_busy.ok = true;
            }
        }
        if (!g_busy.ok || g_busy.running) return;
        g_busy.running = true;
        if (g_busy.last != nil) [g_busy.last waitUntilCompleted];   // (the previous spin has left)
        *(volatile uint32_t*) [g_busy.flag contents] = 0;
        uint32_t cap = std::getenv("DS4_BUSY_CAP") ? (uint32_t) std::atoi(std::getenv("DS4_BUSY_CAP")) : 20000u;   // ~ tens of ms at most
        // the spin's width: one 256-thread threadgroup leaves most of the GPU idle and the clock drops anyway; DS4_BUSY_W
        // threadgroups (256 threads each) keep more of the card's ALUs fed, which is what holds the performance state.
        static const uint32_t width = std::getenv("DS4_BUSY_W") ? (uint32_t) std::max(1, std::atoi(std::getenv("DS4_BUSY_W"))) : 1u;
        id<MTLCommandBuffer> cb = [g_busy.q commandBuffer];
        id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];
        [enc setComputePipelineState:g_busy.ps];
        [enc setBuffer:g_busy.flag offset:0 atIndex:0];
        [enc setBytes:&cap length:4 atIndex:1];
        [enc dispatchThreads:MTLSizeMake(256 * width, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
        [enc endEncoding];
        [cb commit];
        g_busy.last = cb;
    }
}
extern "C" void ds4_busy_end() {
    if (!g_busy.ok || g_busy.last == nil || !g_busy.running) return;
    *(volatile uint32_t*) [g_busy.flag contents] = 1;
    g_busy.running = false;
}

#endif
Context::Context() : impl_(new Impl) {
    impl_->dev = MTLCreateSystemDefaultDevice();
#if defined(STRATA_DS4_RUNTIME)
    if (std::getenv("DS4_TIMELINE") && !g_tl.reg) {
        g_tl.on = true;
        g_tl.reg = true;
        static id<MTLDevice> tl_dev = impl_->dev;
        std::atexit([] { tl_report(tl_dev); });
    }
#endif
    if (!impl_->dev) {
        std::fprintf(stderr, "mtl: no Metal device\n");
        std::exit(2);
    }
    impl_->q = [impl_->dev newCommandQueue];
#if defined(STRATA_DS4_RUNTIME)
    if (ds4_gpu_residency_set && !std::getenv("DS4_NO_RESIDENCY")) {
        id<MTLResidencySet> rs = (__bridge id<MTLResidencySet>)ds4_gpu_residency_set();
        if (rs) [impl_->q addResidencySet:rs];
    }
#endif
    // env, then <exe dir>/metal/ (a component package), then the compiled-in source-tree path
    ggml_common_path = strata_metal::resource("STRATA_GGML_COMMON", "ggml-common.h", STRATA_GGML_COMMON_DEFAULT);
    kernel_dir = strata_metal::resource("STRATA_METAL_KERNELS", "kernels", STRATA_METAL_KERNELS_DEFAULT);
}

Context::~Context() { delete impl_; }

std::string Context::device_name() const { return [[impl_->dev name] UTF8String]; }
bool Context::has_family_apple(int n) const {
    return [impl_->dev supportsFamily:(MTLGPUFamily)(MTLGPUFamilyApple1 + (n - 1))];
}
bool Context::supports_metal4() const {
    if (@available(macOS 26.0, *)) return [impl_->dev supportsFamily:MTLGPUFamilyMetal4];
    return false;
}
uint64_t Context::recommended_working_set() const { return [impl_->dev recommendedMaxWorkingSetSize]; }
uint64_t Context::max_buffer_length() const { return [impl_->dev maxBufferLength]; }

int Context::compile(const std::vector<std::string>& files, const std::string& prelude, bool fast_math) {
    std::string src = prelude + "\n#include <metal_stdlib>\nusing namespace metal;\n";
    if (!ggml_common_path.empty() && prelude.find("NO_GGML_COMMON") == std::string::npos) {
        src += "#define GGML_COMMON_DECL_METAL\n#define GGML_COMMON_IMPL_METAL\n";
        src += read_file(ggml_common_path);
        src += "\n";
    }
    for (const auto& f : files) {
        std::string p = (f.size() && f[0] == '/') ? f : kernel_dir + "/" + f;
        src += "#line 1 \"" + f + "\"\n" + read_file(p) + "\n";
    }
    MTLCompileOptions* o = [MTLCompileOptions new];
    // Safe math by default: no reassociation, so every token of a window is bitwise what a one-token call gives
    // (the kernels write their contractions as explicit fma).  Measured free on the M5 Max.
    fast_math = false;
    if (const char* m = std::getenv("STRATA_METAL_MATH")) fast_math = std::string(m) == "fast";
    if (@available(macOS 15.0, *)) {
        o.mathMode = fast_math ? MTLMathModeFast : MTLMathModeSafe;
    } else {
        o.fastMathEnabled = fast_math;
    }
    NSError* err = nil;
    id<MTLLibrary> lib = [impl_->dev newLibraryWithSource:[NSString stringWithUTF8String:src.c_str()] options:o error:&err];
    if (!lib) {
        std::fprintf(stderr, "mtl: compile failed (%s, macOS %s; %s):\n%s\n", [[impl_->dev name] UTF8String],
                     [[[NSProcessInfo processInfo] operatingSystemVersionString] UTF8String],
                     files.empty() ? "?" : files.back().c_str(), [[err localizedDescription] UTF8String]);
        std::exit(3);
    }
    impl_->libs.push_back(lib);
    return (int)impl_->libs.size() - 1;
}

#if defined(STRATA_DS4_RUNTIME)

// ---- the keep-alive spin (see mtl_rt.h)
static std::atomic<int64_t> g_last_act_ns{0};
static inline int64_t ka_now_ns() { return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
static std::atomic<bool> g_ka_on{false};

void Context::start_keepalive() {
    if (!std::getenv("DS4_KEEPALIVE") || g_ka_on.exchange(true)) return;   // (measured slower in the decode loop)
    id<MTLDevice> dev = impl_->dev;
    std::thread([dev]() {
        @autoreleasepool {
            static const char* kSpin = R"MSL(
#include <metal_stdlib>
using namespace metal;
kernel void ka_spin(device atomic_uint* c [[buffer(0)]], constant uint& n [[buffer(1)]], uint t [[thread_position_in_grid]]) {
    uint x = t;
    for (uint i = 0; i < n; ++i) x = x * 1664525u + 1013904223u;
    if (x == 0xdeadbeefu) atomic_fetch_add_explicit(c, 1u, memory_order_relaxed);
}
)MSL";
            NSError* err = nil;
            id<MTLLibrary> lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kSpin] options:nil error:&err];
            if (!lib) return;
            id<MTLComputePipelineState> ps = [dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"ka_spin"] error:&err];
            if (!ps) return;
            id<MTLCommandQueue> q = [dev newCommandQueue];
            id<MTLBuffer> cbuf = [dev newBufferWithLength:64 options:MTLResourceStorageModeShared];
            uint32_t iters = std::getenv("DS4_KEEPALIVE_ITERS") ? (uint32_t) std::atoi(std::getenv("DS4_KEEPALIVE_ITERS")) : 20000u;
            for (;;) {
                if (ka_now_ns() - g_last_act_ns.load(std::memory_order_relaxed) > 50000000LL) {
                    std::this_thread::sleep_for(std::chrono::milliseconds(2));
                    continue;
                }
                @autoreleasepool {
                    id<MTLCommandBuffer> cb = [q commandBuffer];
                    id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];
                    [enc setComputePipelineState:ps];
                    [enc setBuffer:cbuf offset:0 atIndex:0];
                    [enc setBytes:&iters length:4 atIndex:1];
                    [enc dispatchThreads:MTLSizeMake(256, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
                    [enc endEncoding];
                    [cb commit];
                    [cb waitUntilCompleted];
                }
            }
        }
    }).detach();
}


// ---- zero-copy weights (see mtl_rt.h)
namespace {
struct ZcBuf { size_t len; void* handle; uint64_t addr; };
std::map<uintptr_t, ZcBuf> g_zc;
std::mutex g_zc_mu;
}  // namespace

uint64_t Context::zc_map(const void* p, size_t bytes) {
    const size_t pg = 16384;
    const uintptr_t lo = (uintptr_t) p & ~(uintptr_t) (pg - 1);
    const uintptr_t hi = ((uintptr_t) p + bytes + pg - 1) & ~(uintptr_t) (pg - 1);
    std::lock_guard<std::mutex> lk(g_zc_mu);
    auto it = g_zc.upper_bound(lo);
    if (it != g_zc.begin()) {
        --it;
        if (it->first <= lo && hi <= it->first + it->second.len) return it->second.addr + ((uintptr_t) p - it->first);
    }
    static const size_t cap = std::getenv("DS4_ZC_MAX") ? (size_t) std::atoll(std::getenv("DS4_ZC_MAX")) : 9000;
    if (g_zc.size() >= cap) return 0;
    id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:(void*) lo length:hi - lo options:MTLResourceStorageModeShared deallocator:nil];
    if (!b) return 0;
    if (ds4_gpu_residency_set) {
        id<MTLResidencySet> rs = (__bridge id<MTLResidencySet>)ds4_gpu_residency_set();
        if (rs) {
            [rs addAllocation:b];
            [rs commit];
        }
    }
    ZcBuf z{hi - lo, (__bridge_retained void*) b, (uint64_t) [b gpuAddress]};
    g_zc[lo] = z;
    return z.addr + ((uintptr_t) p - lo);
}
size_t Context::zc_count() const { std::lock_guard<std::mutex> lk(g_zc_mu); return g_zc.size(); }

#endif
Pipeline Context::pipeline(int lib, const std::string& fn, const Constants& c) {
#if defined(STRATA_DS4_RUNTIME)
    // (cached: the host helpers ask for the same pipeline a dispatch at a time; newComputePipelineState costs ~0.3 ms and
    // [MTLDevice name] reads the preferences - the prompt path's expert copies (3 a streamed expert) spent seconds there)
    static std::mutex cache_mu;
    static std::map<std::string, Pipeline> cache;
    std::string key = std::to_string(lib) + "|" + fn;
    for (const auto& v : c.vals) key += "|" + std::to_string(v.index) + ":" + std::to_string(v.type) + ":" + std::to_string(v.v);
    key += "|" + std::to_string((uintptr_t) this);
    {
        std::lock_guard<std::mutex> lk(cache_mu);
        auto it = cache.find(key);
        if (it != cache.end()) return it->second;
    }
    Pipeline made = pipeline_uncached(lib, fn, c);
    {
        std::lock_guard<std::mutex> lk(cache_mu);
        cache[key] = made;
    }
    return made;
}

Pipeline Context::pipeline_uncached(int lib, const std::string& fn, const Constants& c) {
#endif
    NSError* err = nil;
    id<MTLFunction> f;
    NSString* name = [NSString stringWithUTF8String:fn.c_str()];
    if (c.vals.empty()) {
        f = [impl_->libs[lib] newFunctionWithName:name];
        // a function with (optional) function constants needs a specialization even when none is set
        if (f && f.functionConstantsDictionary.count > 0)
            f = [impl_->libs[lib] newFunctionWithName:name constantValues:[MTLFunctionConstantValues new] error:&err];
    } else {
        MTLFunctionConstantValues* cv = [MTLFunctionConstantValues new];
        for (const auto& v : c.vals) {
            int32_t iv = (int32_t)v.v;
            bool bv = v.v != 0;
            uint32_t uv = v.v;
            switch (v.type) {
                case 0: [cv setConstantValue:&iv type:MTLDataTypeInt atIndex:v.index]; break;
                case 1: [cv setConstantValue:&bv type:MTLDataTypeBool atIndex:v.index]; break;
                default: [cv setConstantValue:&uv type:MTLDataTypeUInt atIndex:v.index]; break;
            }
        }
        f = [impl_->libs[lib] newFunctionWithName:name constantValues:cv error:&err];
    }
    if (!f) {
        std::fprintf(stderr, "mtl: no function %s %s\n", fn.c_str(), err ? [[err localizedDescription] UTF8String] : "");
        std::exit(3);
    }
    MTLComputePipelineDescriptor* d = [MTLComputePipelineDescriptor new];
    d.computeFunction = f;
    d.supportIndirectCommandBuffers = YES;
    id<MTLComputePipelineState> ps = [impl_->dev newComputePipelineStateWithDescriptor:d
                                                                               options:MTLPipelineOptionNone
                                                                            reflection:nil
                                                                                 error:&err];
    if (!ps) {
        std::fprintf(stderr, "mtl: pipeline %s: %s\n", fn.c_str(), [[err localizedDescription] UTF8String]);
        std::exit(3);
    }
    Pipeline p;
    p.handle = (__bridge_retained void*)ps;
    p.max_threads = (int)ps.maxTotalThreadsPerThreadgroup;
    if (const char* cap = std::getenv("STRATA_METAL_MAX_THREADS_CAP")) {   // test hook, see require_fit
        const int lim = std::atoi(cap);
        if (lim > 0 && lim < p.max_threads) p.max_threads = lim;
    }
    p.simd_width = (int)ps.threadExecutionWidth;
    p.static_tg_mem = (int)ps.staticThreadgroupMemoryLength;
    p.name = fn;
    if (!c.vals.empty()) {
        p.name += "{";
        for (size_t i = 0; i < c.vals.size(); ++i)
            p.name += (i ? "," : "") + std::to_string(c.vals[i].index) + "=" + std::to_string((int)c.vals[i].v);
        p.name += "}";
    }
#if defined(STRATA_DS4_RUNTIME)
    if (g_device_name == "unknown GPU") g_device_name = [[impl_->dev name] UTF8String];
#else
    g_device_name = [[impl_->dev name] UTF8String];
#endif
    return p;
}

void require_fit(const Pipeline& p, uint32_t threads, const char* where) {
    if (threads == 0 || (int64_t)threads <= (int64_t)p.max_threads) return;
    std::fprintf(stderr,
                 "strata metal: the kernel %s can run at most %d threads per threadgroup on this GPU (%s), but the %s asks "
                 "for %u.\nMetal skips such a dispatch without any error, so every answer would be wrong; stopping here "
                 "instead.  Please report this (the GPU, macOS version and this line) at "
                 "https://github.com/Niko1221/Strata/issues .  (#2323)\n",
                 p.name.empty() ? "?" : p.name.c_str(), p.max_threads, g_device_name.c_str(), where, threads);
    std::fflush(stderr);
    std::exit(6);
}

Buffer Context::alloc(size_t bytes) {
#if defined(STRATA_DS4_RUNTIME)
    // Metal returns the new buffer with one more reference parked in the calling thread's autorelease pool.  The engine's host
    // threads have no pool (only Context::run opens one), so that reference was never given back: every buffer made here lived for
    // the rest of the process, free() or not (a 16 KB job array per expert matvec, ~90K a request: +1.3 GB and a slower queue every
    // call, 16 -> 3 tok/s after a few requests on a big Mac).
    Buffer r;
    @autoreleasepool {
        id<MTLBuffer> b = [impl_->dev newBufferWithLength:bytes options:MTLResourceStorageModeShared];
        if (!b) {
            std::fprintf(stderr, "mtl: alloc %zu failed\n", bytes);
            std::exit(4);
        }
        std::memset([b contents], 0, bytes);
        r.handle = (__bridge_retained void*)b;
        r.ptr = [b contents];
    }
    allocated_ += bytes;
    r.bytes = bytes;
    return r;
#else
    id<MTLBuffer> b = [impl_->dev newBufferWithLength:bytes options:MTLResourceStorageModeShared];
    if (!b) {
        std::fprintf(stderr, "mtl: alloc %zu failed\n", bytes);
        std::exit(4);
    }
    std::memset([b contents], 0, bytes);
    allocated_ += bytes;
    Buffer r;
    r.handle = (__bridge_retained void*)b;
    r.ptr = [b contents];
    r.bytes = bytes;
    return r;
#endif
}

Buffer Context::wrap(void* p, size_t bytes, size_t max_span) {
    const size_t pg = 16384, maxlen = [impl_->dev maxBufferLength] / pg * pg;
    if (bytes <= maxlen || std::getenv("STRATA_METAL_WRAP_SPLIT") && std::string(std::getenv("STRATA_METAL_WRAP_SPLIT")) == "0")
        return wrap(p, bytes);
    const size_t span = (max_span + pg - 1) / pg * pg;
    if (span >= maxlen / 2) {
        std::fprintf(stderr, "mtl: wrap %zu: a tensor of %zu bytes does not fit the chunks\n", bytes, max_span);
        std::exit(4);
    }
    Buffer r;
    r.chunks = std::make_shared<std::vector<void*>>();
    r.stride = maxlen - span;
    for (size_t base = 0; base < bytes; base += r.stride) {
        const size_t len = std::min(maxlen, (bytes - base + pg - 1) / pg * pg);
        id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:(uint8_t*) p + base length:len
                                                        options:MTLResourceStorageModeShared deallocator:nil];
        if (!b) {
            std::fprintf(stderr, "mtl: wrap chunk %zu+%zu failed\n", base, len);
            std::exit(4);
        }
        r.chunks->push_back((__bridge_retained void*)b);
        if (base + len >= bytes) break;
    }
    allocated_ += bytes;
    r.handle = (*r.chunks)[0];
    r.ptr = p;
    r.bytes = bytes;
    std::fprintf(stderr, "mtl: wrapped %.2f GB as %zu overlapping buffers (maxBufferLength %.2f GB, overlap %.2f GB)\n",
                 bytes / 1e9, r.chunks->size(), maxlen / 1e9, span / 1e9);
    return r;
}

#if defined(STRATA_DS4_RUNTIME)
// The runtime's own MTLBuffer under a CPU address (strata::gpu's allocations), when [p, p + bytes) lies inside one.  Binding it
// - the same MTLBuffer, so the same GPU address, as every other kernel that touches this memory - instead of a fresh no-copy
// buffer per call.  (Each fresh wrapper is a second GPU mapping of the same pages: the quantiser read the activations through
// one mapping while the norm that produced them had written through another, and saw the previous layer's.)
extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off) __attribute__((weak_import));
extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu) __attribute__((weak_import));
#endif
Buffer Context::wrap(void* p, size_t bytes) {
#if defined(STRATA_DS4_RUNTIME)
    if (ds4_gpu_buffer_for_cpu && !std::getenv("DS4_WRAP_COPY")) {
        size_t off = 0;
        if (void* mb = ds4_gpu_buffer_for_cpu(p, bytes, &off)) {
            Buffer r;
            r.handle = (__bridge_retained void*)((__bridge id<MTLBuffer>)mb);
            r.ptr = p;
            r.bytes = bytes;
            r.bias = off;
            return r;
        }
    }
#endif
#if defined(STRATA_DS4_RUNTIME)
    if (std::getenv("DS4_WRAP_LOG"))   // debugging: a wrap that is not inside one runtime allocation gets a second GPU mapping of the pages
        std::fprintf(stderr, "mtl: wrap FALLBACK p=%p bytes=%zu alloc_remaining=%zu\n", p, bytes, ds4_gpu_alloc_remaining ? ds4_gpu_alloc_remaining(p) : (size_t) 0);
#endif
#if defined(STRATA_DS4_RUNTIME)
    Buffer r;
    @autoreleasepool {   // (see alloc: the engine's host threads have no pool of their own)
        id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];
        if (!b) {
            std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);
            std::exit(4);
        }
        r.handle = (__bridge_retained void*)b;
    }
    allocated_ += bytes;
    r.ptr = p;
    r.bytes = bytes;
    return r;
#else
    id<MTLBuffer> b = [impl_->dev newBufferWithBytesNoCopy:p length:bytes options:MTLResourceStorageModeShared deallocator:nil];
    if (!b) {
        std::fprintf(stderr, "mtl: wrap %zu failed\n", bytes);
        std::exit(4);
    }
    allocated_ += bytes;
    Buffer r;
    r.handle = (__bridge_retained void*)b;
    r.ptr = p;
    r.bytes = bytes;
    return r;
#endif
}

void Context::free(Buffer& b) {
#if defined(STRATA_DS4_RUNTIME)
    // The deferred command buffer is made WITHOUT retained references: a buffer released here while a dispatch that names it
    // is still queued is read after it is gone (or after another alloc took its memory: the job arrays of iq_mv read the
    // NEXT call's jobs, measured as rows of zeros and different tokens every run).  Hold it until the buffer has completed.
    std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
    const bool hold = impl_->cb_def != nil || impl_->inflight != nil;
#endif
    if (b.chunks) {
#if defined(STRATA_DS4_RUNTIME)
        for (void* h : *b.chunks) {
            if (hold) impl_->graveyard.push_back(h);
            else { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
        }
#else
        for (void* h : *b.chunks) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
#endif
        b = Buffer{};
        return;
    }
    if (b.handle) {
#if defined(STRATA_DS4_RUNTIME)
        if (hold) impl_->graveyard.push_back(b.handle);
        else { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)b.handle; x = nil; }
#else
        id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)b.handle;
        x = nil;
#endif
    }
    b = Buffer{};
}

#if defined(STRATA_DS4_RUNTIME)
static std::mutex g_hist_mu;
static std::unordered_map<void*, std::string> g_pipe_names;
static std::unordered_map<std::string, uint64_t> g_kernel_counts;
// DS4_ENC_TIMES: where a dispatch's encoding time goes - the pipeline state, the arguments, the dispatch call
static std::atomic<uint64_t> g_t_pipe{0}, g_t_args{0}, g_t_disp{0};
#endif
static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
    require_fit(d, "dispatch");
#if defined(STRATA_DS4_RUNTIME)
    static const bool enc_times = std::getenv("DS4_ENC_TIMES") != nullptr;
    const auto t_a = std::chrono::steady_clock::now();
#endif
    [enc setComputePipelineState:P(d.p)];
#if defined(STRATA_DS4_RUNTIME)
    const auto t_b = std::chrono::steady_clock::now();
#endif
    for (size_t i = 0; i < d.args.size(); ++i) {
        const Arg& a = d.args[i];
        if (a.buf) {
            size_t o;
            id<MTLBuffer> mb = BO(*a.buf, a.off, o);
#if defined(STRATA_DS4_RUNTIME)
            static const bool chkbuf = std::getenv("DS4_CHECK_BUF") != nullptr;
            if (chkbuf && o >= [mb length]) std::fprintf(stderr, "mtl: %s arg %zu: offset %zu past the buffer (%zu bytes; Buffer.bytes %zu bias %zu off %zu)\n", d.p->name.c_str(), i, o, (size_t) [mb length], a.buf->bytes, a.buf->bias, (size_t) a.off);
#endif
            [enc setBuffer:mb offset:o atIndex:i];
        }
        else
            [enc setBytes:a.bytes.data() length:a.bytes.size() atIndex:i];
    }
#if defined(STRATA_DS4_RUNTIME)
    const auto t_c = std::chrono::steady_clock::now();
#endif
    if (d.tg_mem) [enc setThreadgroupMemoryLength:(d.tg_mem + 15) / 16 * 16 atIndex:0];
    if (d.indirect)
        [enc dispatchThreadgroupsWithIndirectBuffer:B(*d.indirect)
                               indirectBufferOffset:d.ind_off
                              threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];
    else
        [enc dispatchThreadgroups:MTLSizeMake(d.grid[0], d.grid[1], d.grid[2])
            threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];
#if defined(STRATA_DS4_RUNTIME)
    if (enc_times) {
        const auto t_d = std::chrono::steady_clock::now();
        g_t_pipe.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_b - t_a).count(), std::memory_order_relaxed);
        g_t_args.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_c - t_b).count(), std::memory_order_relaxed);
        g_t_disp.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_d - t_c).count(), std::memory_order_relaxed);
    }
}

static std::deque<std::string> g_recent;   // the last dispatches encoded (printed with a command buffer failure)
static std::string recent_str() { std::string r; for (auto& n : g_recent) r += n + " "; return r; }
static double g_gpu_ms = 0.0;
static std::atomic<uint64_t> g_gpu_busy_us{0};   // DS4_GPU_TOTAL: the GPU time of every committed command buffer
static void add_busy_handler(id<MTLCommandBuffer> cb) {
    static const bool on = std::getenv("DS4_GPU_TOTAL") != nullptr;
    if (on) [cb addCompletedHandler:^(id<MTLCommandBuffer> c) { const double d = c.GPUEndTime - c.GPUStartTime; if (d > 0) g_gpu_busy_us += (uint64_t) (d * 1e6); }];
}   // GPU time of every committed command buffer, summed
static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0};   // host time: waiting on command buffers; encoding dispatches
static std::mutex g_prof_mu;
static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof;   // DS4_PROF: kernel -> (GPU us, dispatches)
static std::string g_prof_last;
static std::atomic<uint64_t> g_cb_count{0};
static bool g_count_reg = false;
static void ds4_report_counts() {
    if (g_gpu_busy_us.load()) std::fprintf(stderr, "mtl: GPU busy (every command buffer) %.1f ms\n", g_gpu_busy_us.load() / 1000.0);
    std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,
                 (unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());
    std::fprintf(stderr, "mtl: host waited %.1f ms for the GPU in %llu waits; encoding the dispatches took %.1f ms\n", g_wait_ns.load() / 1e6, (unsigned long long) g_wait_n.load(), g_enc_ns.load() / 1e6);
    if (g_t_pipe.load() || g_t_args.load() || g_t_disp.load())
        std::fprintf(stderr, "mtl: of the encoding: pipeline %.1f ms, arguments %.1f ms, dispatch %.1f ms\n",
                     g_t_pipe.load() / 1e6, g_t_args.load() / 1e6, g_t_disp.load() / 1e6);
    if (!g_prof.empty()) {
        std::vector<std::pair<double, std::string>> pv;
        double tot = 0;
        for (auto& kv : g_prof) { pv.emplace_back(kv.second.first, kv.first); tot += kv.second.first; }
        std::sort(pv.begin(), pv.end(), [](auto& a, auto& b) { return a.first > b.first; });
        std::fprintf(stderr, "prof: kernel                          GPU ms    n   us/dispatch   (total %.1f ms)\n", tot / 1000.0);
        for (size_t i = 0; i < pv.size() && i < 30; ++i) {
            auto& e = g_prof[pv[i].second];
            std::fprintf(stderr, "prof: %-30s %8.1f %6llu %8.1f\n", pv[i].second.c_str(), e.first / 1000.0, (unsigned long long) e.second, e.first / (double) e.second);
        }
    }
    std::vector<std::pair<uint64_t, std::string>> v;
    { std::lock_guard<std::mutex> lk(g_hist_mu); for (auto& kv : g_kernel_counts) v.emplace_back(kv.second, kv.first); }
    std::sort(v.begin(), v.end(), [](auto& a, auto& b) { return a.first > b.first; });
    for (size_t i = 0; i < v.size() && i < 14; ++i)
        std::fprintf(stderr, "  %6llu  %s\n", (unsigned long long) v[i].first, v[i].second.c_str());
}

extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }

extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));
void Context::split_runs() {
    // (the streams' pending copies come first, as in a full flush: they are queued on another MTLCommandQueue and would
    // otherwise not have run when the commands committed here read what they write)
    if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();
    // Commit the batched dispatches as their own command buffer WITHOUT waiting: the queue runs buffers in order and makes each
    // one's writes visible to the next.  (Measured: a producer kernel's output was read stale by the next dispatch of the same
    // buffer for two kernels in a layer; ending the buffer after them is enough, no wait needed.)
    std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
    if (impl_->cb_def == nil) return;
    ++g_cb_count;
    [impl_->enc_def endEncoding];
    if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];
    [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
    add_busy_handler(impl_->cb_def);
    [impl_->cb_def commit];
    impl_->inflight = impl_->cb_def;
    impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();
    impl_->cb_def = nil;
    impl_->enc_def = nil;
    impl_->def_n = 0;
}

// Waiting for a committed command buffer: spin on its status for a while (the thread wake of waitUntilCompleted costs
// ~1 ms a wait; the decode token waits ~100 times), then block.  DS4_NO_SPIN=1: always block.
static void wait_spin(id<MTLCommandBuffer> cb) {
    static const bool no_spin = std::getenv("DS4_NO_SPIN") != nullptr;
    static const double budget = std::getenv("DS4_SPIN_US") ? std::atof(std::getenv("DS4_SPIN_US")) * 1e-6 : 20e-3;
    if (!no_spin) {
        const auto t0 = std::chrono::steady_clock::now();
        while (cb.status < MTLCommandBufferStatusCompleted) {
            if (std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() > budget) break;
        }
    }
    if (cb.status < MTLCommandBufferStatusCompleted) [cb waitUntilCompleted];
}

void* Context::mark() {
    std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
    if (impl_->cb_def != nil) split_runs();   // (the streams' pending work goes first, inside)
    if (impl_->inflight == nil) return nullptr;
    return (__bridge_retained void*) impl_->inflight;
}

void Context::drop_mark(void* token) {
    // The token holds a +1 on a committed command buffer; one nobody waits for is never freed.
    if (token) { id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token; cb = nil; }
}

void Context::wait_mark(void* token) {
    id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;
    const auto tw0 = std::chrono::steady_clock::now();
    wait_spin(cb);
    g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();
    ++g_wait_n;
    if (cb.status != MTLCommandBufferStatusCompleted)
        std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);
    if (cb.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: recent: %s\n", recent_str().c_str()), std::fprintf(stderr, "mtl: info: %s\n", [[[cb.error userInfo] description] UTF8String]);
}

void Context::flush_runs() {
    if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);
    static const bool cb_log = std::getenv("DS4_CB_LOG") != nullptr;   // debugging: one line a command buffer (what it held)
    if (cb_log && impl_->cb_def != nil) {
        std::string first, last;
        if (!impl_->def_names.empty()) {
            size_t sp = impl_->def_names.find(' ');
            first = impl_->def_names.substr(0, sp);
            size_t e = impl_->def_names.find_last_not_of(' ');
            size_t b = impl_->def_names.find_last_of(' ', e);
            last = impl_->def_names.substr(b == std::string::npos ? 0 : b + 1, e - (b == std::string::npos ? 0 : b));
        }
        std::fprintf(stderr, "CB %d disp  first=%-24s last=%s\n", impl_->def_n, first.c_str(), last.c_str());
    }
    std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
    if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }
    if (impl_->cb_def != nil) {
        ++g_cb_count;
        [impl_->enc_def endEncoding];
        if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];
        [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];
    add_busy_handler(impl_->cb_def);
        [impl_->cb_def commit];
        impl_->inflight = impl_->cb_def;
        impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();
        impl_->cb_def = nil;
        impl_->enc_def = nil;
        impl_->def_n = 0;
    }
    if (impl_->inflight != nil) {
        if (std::getenv("DS4_TRACE_WAIT")) { void* fr[8]; const int nf = backtrace(fr, 8); std::fprintf(stderr, "WAIT flush_runs:\n"); backtrace_symbols_fd(fr + 1, nf - 1, 2); }
        const auto tw0 = std::chrono::steady_clock::now();
        wait_spin(impl_->inflight);
        g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();
        ++g_wait_n;
        { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;
          if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }
        if (impl_->inflight.status != MTLCommandBufferStatusCompleted)
            std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);
        if (impl_->inflight.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: failed cb held: %s\n", impl_->inflight_names.c_str());
        impl_->inflight = nil;
    }
    for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }
    impl_->graveyard.clear();
#endif
}

double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {
#if defined(STRATA_DS4_RUNTIME)
    std::lock_guard<std::recursive_mutex> lk_(impl_->mu);
#endif
    @autoreleasepool {
#if defined(STRATA_DS4_RUNTIME)
        if (ds.size() == 1 && reps == 1 && !barrier_between) {
            { static const std::string skipk = std::getenv("DS4_SKIP_K") ? std::string(",") + std::getenv("DS4_SKIP_K") + "," : std::string();   // debugging: drop these kernels (timing only)
              if (!skipk.empty() && skipk.find("," + ds[0].p->name + ",") != std::string::npos) return 0.0; }
            if (impl_->cb_def == nil || impl_->def_n >= 2000000) {
                if (impl_->cb_def != nil) split_runs();   // (a buffer committed earlier may still be running: the queue keeps the order)
                if (std::getenv("DS4_ERRINFO")) {
                    MTLCommandBufferDescriptor* cd = [MTLCommandBufferDescriptor new];
                    cd.retainedReferences = NO;
                    cd.errorOptions = MTLCommandBufferErrorOptionEncoderExecutionStatus;
                    impl_->cb_def = [impl_->q commandBufferWithDescriptor:cd];
                } else
                impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];
                // (the wait on our own event is only for a *different* queue: this queue already runs the previous buffer
                // before this one, and the wait costs a full GPU round trip at every buffer - 107 buffers a decode token,
                // 0.28 ms each.  DS4_QEV_WAIT=1 restores it.)
                static const bool qev_wait = std::getenv("DS4_NO_QEV_WAIT") == nullptr;   // (measured: off is slower - keep it)
                if (qev_wait && impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];
                impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial);
            }
            const auto te0 = std::chrono::steady_clock::now();
            encode(impl_->enc_def, ds[0]);
            if (g_tl.on && g_tl.att && g_tl.n < g_tl.cap) { g_tl.names[g_tl.n++] = ds[0].p->name; g_tl.att = false; }
            g_last_act_ns.store(ka_now_ns(), std::memory_order_relaxed);
            g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();
            if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";
            g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();
            static const bool trace = std::getenv("DS4_TRACE_DISP") != nullptr;
            if (trace) { std::lock_guard<std::mutex> lk(g_hist_mu); std::fprintf(stderr, "TRACE disp %s\n", ds[0].p->name.c_str()); }
            // Every wrapper binds its own MTLBuffer over the same memory, so Metal's hazard tracking cannot see that one
            // dispatch reads what the previous wrote: without an explicit barrier the reader saw the stale value (measured: the
            // second matvec's activation quantiser read the previous layer's x).
            static const bool no_mb = std::getenv("DS4_NO_MBARRIER") != nullptr;
            if (!no_mb) [impl_->enc_def memoryBarrierWithScope:MTLBarrierScopeBuffers];
            static const bool enc_each = std::getenv("DS4_ENC_EACH") != nullptr || g_tl.on;   // a new encoder per dispatch (debugging, the timeline)
            if (enc_each) {
                [impl_->enc_def endEncoding];
                impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, (std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial));
            }
            ++impl_->def_n;
            ++g_disp_count;
            static const bool prof = std::getenv("DS4_PROF") != nullptr;   // every dispatch alone, its GPU time kept per kernel
            if (prof) { g_prof_last = ds[0].p->name; if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak(); flush_runs(); return 0.0; }
            static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr;   // debugging: every dispatch alone, in order
            static const std::string only = std::getenv("DS4_FLUSH_AFTER") ? std::getenv("DS4_FLUSH_AFTER") : "";   // ",name1,name2,"
            const bool listed = !only.empty() && only.find("," + ds[0].p->name + ",") != std::string::npos;
            if (each || listed) {
                if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();
                flush_runs();
            }
            return 0.0;
        }
        flush_runs();
#endif
        id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];
        id<MTLComputeCommandEncoder> enc =
            [cb computeCommandEncoderWithDispatchType:concurrent ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];
        for (int r = 0; r < reps; ++r)
            for (const auto& d : ds) {
                encode(enc, d);
#if defined(STRATA_DS4_RUNTIME)
                ++g_disp_count;
#endif
                if (barrier_between) [enc memoryBarrierWithScope:MTLBarrierScopeBuffers];
            }
        [enc endEncoding];
        [cb commit];
        [cb waitUntilCompleted];
        if (cb.status != MTLCommandBufferStatusCompleted) {
            std::fprintf(stderr, "mtl: command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);
            std::exit(5);
        }
        return cb.GPUEndTime - cb.GPUStartTime;
    }
}

double Context::run_graph(const std::vector<Dispatch>& ds) {
    @autoreleasepool {
        id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];
        id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoderWithDispatchType:MTLDispatchTypeConcurrent];
        for (size_t i = 0; i < ds.size(); ++i) {
            if (i > 0 && ds[i].barrier) [enc memoryBarrierWithScope:MTLBarrierScopeBuffers];
            encode(enc, ds[i]);
        }
        [enc endEncoding];
        [cb commit];
        [cb waitUntilCompleted];
        if (cb.status != MTLCommandBufferStatusCompleted) {
            std::fprintf(stderr, "mtl: command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);
            std::exit(5);
        }
        return cb.GPUEndTime - cb.GPUStartTime;
    }
}

double Context::run_graph_seg(const std::vector<Dispatch>& ds, const std::vector<int>& cuts, void (*at_cut)(void*, int),
                              void* user) {
    @autoreleasepool {
        if (impl_->ev == nil) impl_->ev = [impl_->dev newSharedEvent];
        id<MTLSharedEvent> ev = impl_->ev;
        const uint64_t base = impl_->ev_v;
        id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];
        size_t i = 0;
        for (size_t s = 0; s <= cuts.size(); ++s) {
            const size_t end = s < cuts.size() ? (size_t) cuts[s] : ds.size();
            id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoderWithDispatchType:MTLDispatchTypeConcurrent];
            const size_t first = i;
            for (; i < end; ++i) {
                if (i > first && ds[i].barrier) [enc memoryBarrierWithScope:MTLBarrierScopeBuffers];
                encode(enc, ds[i]);
            }
            [enc endEncoding];
            if (s < cuts.size()) {
                [cb encodeSignalEvent:ev value:base + 2 * s + 1];
                [cb encodeWaitForEvent:ev value:base + 2 * s + 2];
            }
        }
        [cb commit];
        for (size_t s = 0; s < cuts.size(); ++s) {
            const uint64_t want = base + 2 * s + 1;
            while (ev.signaledValue < want) {
                if (cb.status >= MTLCommandBufferStatusCompleted) break;   // an error: the wait below reports it
            }
            at_cut(user, (int) s);
            ev.signaledValue = base + 2 * s + 2;
        }
        impl_->ev_v = base + 2 * cuts.size() + 2;
        [cb waitUntilCompleted];
        if (cb.status != MTLCommandBufferStatusCompleted) {
            std::fprintf(stderr, "mtl: command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);
            std::exit(5);
        }
        return cb.GPUEndTime - cb.GPUStartTime;
    }
}

double Context::run_wall(const std::vector<Dispatch>& ds, int reps, double* gpu_seconds) {
    double gpu = 0;
    const double t0 = now_s();
    for (int r = 0; r < reps; ++r) {
        @autoreleasepool {
            id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];
            id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];
            for (const auto& d : ds) encode(enc, d);
            [enc endEncoding];
            [cb commit];
            [cb waitUntilCompleted];
            gpu += cb.GPUEndTime - cb.GPUStartTime;
        }
    }
    if (gpu_seconds) *gpu_seconds = gpu;
    return now_s() - t0;
}

void Context::warm(const std::vector<Dispatch>& ds, double seconds) {
    const double t0 = now_s();
    while (now_s() - t0 < seconds) run(ds);
}

double Context::run_icb(const std::vector<Dispatch>& ds, int reps, bool barrier_between) {
    @autoreleasepool {
        size_t maxb = 1;
        for (const auto& d : ds) maxb = std::max(maxb, d.args.size());
        MTLIndirectCommandBufferDescriptor* desc = [MTLIndirectCommandBufferDescriptor new];
        desc.commandTypes = MTLIndirectCommandTypeConcurrentDispatch;
        desc.inheritBuffers = NO;
        desc.inheritPipelineState = NO;
        desc.maxKernelBufferBindCount = maxb;
        id<MTLIndirectCommandBuffer> icb = [impl_->dev newIndirectCommandBufferWithDescriptor:desc
                                                                              maxCommandCount:ds.size()
                                                                                      options:0];
        NSMutableSet* used = [NSMutableSet new];
        for (size_t i = 0; i < ds.size(); ++i) {
            const Dispatch& d = ds[i];
            require_fit(d, "indirect-command dispatch");
            id<MTLIndirectComputeCommand> c = [icb indirectComputeCommandAtIndex:i];
            [c setComputePipelineState:P(d.p)];
            for (size_t a = 0; a < d.args.size(); ++a) {
                if (!d.args[a].buf) {
                    std::fprintf(stderr, "mtl: run_icb needs buffer arguments only\n");
                    std::exit(5);
                }
                size_t o;
                id<MTLBuffer> mb = BO(*d.args[a].buf, d.args[a].off, o);
                [c setKernelBuffer:mb offset:o atIndex:a];
                [used addObject:mb];
            }
            if (d.tg_mem) [c setThreadgroupMemoryLength:(d.tg_mem + 15) / 16 * 16 atIndex:0];
            [c concurrentDispatchThreadgroups:MTLSizeMake(d.grid[0], d.grid[1], d.grid[2])
                        threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];
            if (barrier_between) [c setBarrier];
        }
        id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];
        id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoderWithDispatchType:MTLDispatchTypeConcurrent];
        for (id<MTLBuffer> b in used) [enc useResource:b usage:MTLResourceUsageRead | MTLResourceUsageWrite];
        for (int r = 0; r < reps; ++r) {
            [enc executeCommandsInBuffer:icb withRange:NSMakeRange(0, ds.size())];
            if (barrier_between) [enc memoryBarrierWithScope:MTLBarrierScopeBuffers];
        }
        [enc endEncoding];
        [cb commit];
        [cb waitUntilCompleted];
        if (cb.status != MTLCommandBufferStatusCompleted) {
            std::fprintf(stderr, "mtl: icb command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);
            std::exit(5);
        }
        return cb.GPUEndTime - cb.GPUStartTime;
    }
}

}  // namespace mtl