Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
14.2 KB
Raw Blame
// mtl_probe.cpp - M5 Max checks for the Metal device-interface design (mac-design stream).
//  1. device facts: unified memory, maxBufferLength, recommendedMaxWorkingSetSize, GPU families
//  2. MSL compiled at run time (no offline `metal` tool in the Command Line Tools)
//  3. a hello kernel that takes RAW GPU ADDRESSES in a setBytes struct (the CUDA "void* device pointer" model)
//  4. gpuAddress vs contents(): are they equal?
//  5. newBufferWithBytesNoCopy over an mmapped file > 30 GB: alignment, max length, split into chunks, GPU reads it
// Build: clang++ -std=c++20 -O2 -I metal-cpp mtl_probe.cpp -framework Metal -framework Foundation -framework QuartzCore
#define NS_PRIVATE_IMPLEMENTATION
#define MTL_PRIVATE_IMPLEMENTATION
#include <Foundation/Foundation.hpp>
#include <Metal/Metal.hpp>

#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>

#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <vector>

static double now_ms() {
    return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
}

static const char* kSrc = R"MSL(
#include <metal_stdlib>
using namespace metal;
struct AxpyArgs { device const float* x; device float* y; float a; uint n; };
kernel void axpy(constant AxpyArgs& p [[buffer(0)]], uint i [[thread_position_in_grid]]) {
    if (i < p.n) p.y[i] = p.a * p.x[i] + p.y[i];
}
// reads one uint32 at each probe offset of a (possibly huge) buffer given by raw GPU address
struct ProbeArgs { device const uint* base; device const ulong* offs; device uint* out; uint n; };
kernel void probe(constant ProbeArgs& p [[buffer(0)]], uint i [[thread_position_in_grid]]) {
    if (i < p.n) p.out[i] = p.base[p.offs[i] / 4];
}
// streaming read of a buffer range: sum of uint32 (bandwidth through a NoCopy file buffer)
struct SumArgs { device const uint4* base; device atomic_uint* out; ulong n4; };
kernel void sum(constant SumArgs& p [[buffer(0)]], uint i [[thread_position_in_grid]], uint g [[threads_per_grid]]) {
    uint s = 0;
    for (ulong k = i; k < p.n4; k += g) { uint4 v = p.base[k]; s += v.x ^ v.y ^ v.z ^ v.w; }
    atomic_fetch_add_explicit(p.out, s, memory_order_relaxed);
}
)MSL";

struct AxpyArgs { uint64_t x, y; float a; uint32_t n; };
struct ProbeArgs { uint64_t base, offs, out; uint32_t n; };
struct SumArgs { uint64_t base, out; uint64_t n4; };

int main(int argc, char** argv) {
    NS::AutoreleasePool* pool = NS::AutoreleasePool::alloc()->init();
    MTL::Device* dev = MTL::CreateSystemDefaultDevice();
    if (!dev) { std::printf("no Metal device\n"); return 1; }
    std::printf("device: %s\n", dev->name()->utf8String());
    std::printf("unified memory: %d\n", (int) dev->hasUnifiedMemory());
    std::printf("maxBufferLength: %llu (%.2f GiB)\n", (unsigned long long) dev->maxBufferLength(), dev->maxBufferLength() / 1073741824.0);
    std::printf("recommendedMaxWorkingSetSize: %llu (%.2f GiB)\n", (unsigned long long) dev->recommendedMaxWorkingSetSize(),
                dev->recommendedMaxWorkingSetSize() / 1073741824.0);
    std::printf("maxThreadgroupMemoryLength: %llu\n", (unsigned long long) dev->maxThreadgroupMemoryLength());
    std::printf("families: apple7 %d apple8 %d apple9 %d apple10 %d metal3 %d metal4 %d\n", (int) dev->supportsFamily(MTL::GPUFamilyApple7),
                (int) dev->supportsFamily(MTL::GPUFamilyApple8), (int) dev->supportsFamily(MTL::GPUFamilyApple9),
                (int) dev->supportsFamily(MTL::GPUFamilyApple10), (int) dev->supportsFamily(MTL::GPUFamilyMetal3),
                (int) dev->supportsFamily(MTL::GPUFamilyMetal4));
    std::printf("page size: %ld\n", sysconf(_SC_PAGESIZE));

    // ---- 2. run-time MSL compile
    NS::Error* e = nullptr;
    double t0 = now_ms();
    MTL::CompileOptions* co = MTL::CompileOptions::alloc()->init();
    MTL::Library* lib = dev->newLibrary(NS::String::string(kSrc, NS::UTF8StringEncoding), co, &e);
    if (!lib) { std::printf("MSL compile failed: %s\n", e ? e->localizedDescription()->utf8String() : "?"); return 1; }
    std::printf("MSL run-time compile: %.1f ms\n", now_ms() - t0);
    auto pipe = [&](const char* n) {
        MTL::Function* f = lib->newFunction(NS::String::string(n, NS::UTF8StringEncoding));
        MTL::ComputePipelineState* p = dev->newComputePipelineState(f, &e);
        if (!p) { std::printf("pipeline %s failed\n", n); std::exit(1); }
        return p;
    };
    MTL::ComputePipelineState* p_axpy = pipe("axpy");
    MTL::ComputePipelineState* p_probe = pipe("probe");
    MTL::ComputePipelineState* p_sum = pipe("sum");
    MTL::CommandQueue* q = dev->newCommandQueue();

    // ---- 3/4. hello kernel through raw GPU addresses; residency through a residency set
    const uint32_t N = 1 << 20;
    MTL::Buffer* bx = dev->newBuffer(N * 4, MTL::ResourceStorageModeShared);
    MTL::Buffer* by = dev->newBuffer(N * 4, MTL::ResourceStorageModeShared);
    std::printf("buffer x: contents %p gpuAddress 0x%llx (%s)\n", bx->contents(), (unsigned long long) bx->gpuAddress(),
                (uint64_t) bx->contents() == bx->gpuAddress() ? "EQUAL" : "different");
    float* hx = (float*) bx->contents();
    float* hy = (float*) by->contents();
    for (uint32_t i = 0; i < N; ++i) { hx[i] = (float) i; hy[i] = 1.0f; }
    MTL::ResidencySetDescriptor* rd = MTL::ResidencySetDescriptor::alloc()->init();
    MTL::ResidencySet* rs = dev->newResidencySet(rd, &e);
    if (!rs) { std::printf("residency set failed\n"); return 1; }
    rs->addAllocation(bx); rs->addAllocation(by); rs->commit();
    q->addResidencySet(rs);
    {
        AxpyArgs a{bx->gpuAddress(), by->gpuAddress(), 2.0f, N};
        MTL::CommandBuffer* cb = q->commandBuffer();
        MTL::ComputeCommandEncoder* enc = cb->computeCommandEncoder();
        enc->setComputePipelineState(p_axpy);
        enc->setBytes(&a, sizeof a, 0);
        enc->dispatchThreads(MTL::Size(N, 1, 1), MTL::Size(256, 1, 1));
        enc->endEncoding();
        cb->commit();
        cb->waitUntilCompleted();
        int bad = 0;
        for (uint32_t i = 0; i < N; ++i) if (hy[i] != 2.0f * i + 1.0f) ++bad;
        std::printf("hello axpy via raw GPU addresses + residency set: %s (%d bad)\n", bad ? "FAIL" : "OK", bad);
    }
    // MTLSharedEvent as the CUDA event: signal from the GPU, poll from the host
    {
        MTL::SharedEvent* ev = dev->newSharedEvent();
        MTL::CommandBuffer* cb = q->commandBuffer();
        MTL::ComputeCommandEncoder* enc = cb->computeCommandEncoder();
        AxpyArgs a{bx->gpuAddress(), by->gpuAddress(), 1.0f, N};
        enc->setComputePipelineState(p_axpy);
        enc->setBytes(&a, sizeof a, 0);
        enc->dispatchThreads(MTL::Size(N, 1, 1), MTL::Size(256, 1, 1));
        enc->endEncoding();
        cb->encodeSignalEvent(ev, 1);
        double ts = now_ms();
        cb->commit();
        long spins = 0;
        while (ev->signaledValue() < 1) ++spins;
        std::printf("shared event: signalled after %.3f ms, %ld host polls\n", now_ms() - ts, spins);
        ev->release();
    }
    // dispatch cost: 1,000 tiny dispatches in one command buffer (the window-replay cost model)
    {
        AxpyArgs a{bx->gpuAddress(), by->gpuAddress(), 0.0f, 256};
        for (int rep = 0; rep < 3; ++rep) {
            double ts = now_ms();
            MTL::CommandBuffer* cb = q->commandBufferWithUnretainedReferences();
            MTL::ComputeCommandEncoder* enc = cb->computeCommandEncoder();
            for (int k = 0; k < 1000; ++k) {
                enc->setComputePipelineState(p_axpy);
                enc->setBytes(&a, sizeof a, 0);
                enc->dispatchThreads(MTL::Size(256, 1, 1), MTL::Size(256, 1, 1));
            }
            enc->endEncoding();
            const double te = now_ms();
            cb->commit();
            cb->waitUntilCompleted();
            std::printf("1000 dependent tiny dispatches: encode %.3f ms, encode+run %.3f ms\n", te - ts, now_ms() - ts);
        }
    }

    // ---- 5. a > 30 GB mmapped file wrapped with newBufferWithBytesNoCopy
    if (argc < 2) { pool->release(); return 0; }
    const char* path = argv[1];
    const int fd = open(path, O_RDONLY);
    if (fd < 0) { std::printf("open %s failed\n", path); return 1; }
    struct stat sb; fstat(fd, &sb);
    const uint64_t fsize = (uint64_t) sb.st_size;
    const uint64_t pg = (uint64_t) sysconf(_SC_PAGESIZE);
    const uint64_t mlen = (fsize + pg - 1) / pg * pg;   // NoCopy wants a page-multiple length
    std::printf("file %s: %.2f GiB\n", path, fsize / 1073741824.0);
    for (int mode = 0; mode < 2; ++mode) {
        // mode 0: PROT_READ MAP_SHARED (read-only, what the engine wants); mode 1: PROT_READ|PROT_WRITE MAP_PRIVATE
        const int prot = mode == 0 ? PROT_READ : (PROT_READ | PROT_WRITE);
        const int flags = mode == 0 ? MAP_SHARED : MAP_PRIVATE;
        void* m = mmap(nullptr, mlen, prot, flags, fd, 0);
        if (m == MAP_FAILED) { std::printf("mmap mode %d failed\n", mode); continue; }
        std::printf("-- mode %d (%s): mmap at %p (page aligned %d)\n", mode, mode == 0 ? "PROT_READ MAP_SHARED" : "RW MAP_PRIVATE", m,
                    (int) (((uintptr_t) m % pg) == 0));
        double ts = now_ms();
        MTL::Buffer* whole = dev->newBuffer(m, mlen, MTL::ResourceStorageModeShared, nullptr);
        std::printf("   one NoCopy buffer of %.2f GiB: %s (%.1f ms)\n", mlen / 1073741824.0, whole ? "OK" : "nil", now_ms() - ts);
        // chunks of at most maxBufferLength / 2 (and a 4 GiB variant: one chunk per few layers)
        std::vector<MTL::Buffer*> chunks;
        const uint64_t chunk = argc > 2 ? (std::strtoull(argv[2], nullptr, 10) << 20) : (4ull << 30);
        ts = now_ms();
        bool all = true;
        for (uint64_t off = 0; off < mlen; off += chunk) {
            const uint64_t len = std::min(chunk, mlen - off);
            MTL::Buffer* b = dev->newBuffer((char*) m + off, len, MTL::ResourceStorageModeShared, nullptr);
            if (!b) { all = false; break; }
            chunks.push_back(b);
        }
        std::printf("   %zu NoCopy chunks of %.2f GiB: %s (%.1f ms)\n", chunks.size(), chunk / 1073741824.0, all ? "OK" : "FAILED", now_ms() - ts);
        // GPU probe reads at 64 offsets spread over the file (the file's pattern: uint32 at offset o equals (o>>12) ^ 0x5a5a5a5a
        // where written; holes read 0)
        const uint32_t NP = 64;
        MTL::Buffer* boffs = dev->newBuffer(NP * 8, MTL::ResourceStorageModeShared);
        MTL::Buffer* bout = dev->newBuffer(NP * 4, MTL::ResourceStorageModeShared);
        uint64_t* offs = (uint64_t*) boffs->contents();
        for (uint32_t i = 0; i < NP; ++i) offs[i] = ((fsize / NP) * i) & ~(uint64_t) 4095;
        auto run_probe = [&](MTL::Buffer* b, uint64_t base_off, const char* what) {
            // offsets relative to this buffer's base
            std::vector<uint64_t> keep(offs, offs + NP);
            uint32_t n = 0;
            std::vector<uint32_t> idx;
            for (uint32_t i = 0; i < NP; ++i)
                if (keep[i] >= base_off && keep[i] < base_off + b->length()) { offs[n++] = keep[i] - base_off; idx.push_back(i); }
            MTL::ResidencySet* r2 = dev->newResidencySet(rd, &e);
            r2->addAllocation(b); r2->addAllocation(boffs); r2->addAllocation(bout); r2->commit();
            double tr = now_ms();
            r2->requestResidency();
            const double treq = now_ms() - tr;
            ProbeArgs a{b->gpuAddress(), boffs->gpuAddress(), bout->gpuAddress(), n};
            MTL::CommandBuffer* cb = q->commandBuffer();
            cb->useResidencySet(r2);
            MTL::ComputeCommandEncoder* enc = cb->computeCommandEncoder();
            enc->setComputePipelineState(p_probe);
            enc->setBytes(&a, sizeof a, 0);
            enc->dispatchThreads(MTL::Size(n ? n : 1, 1, 1), MTL::Size(64, 1, 1));
            enc->endEncoding();
            tr = now_ms();
            cb->commit();
            cb->waitUntilCompleted();
            const double trun = now_ms() - tr;
            int okc = 0, bad = 0;
            const uint32_t* out = (const uint32_t*) bout->contents();
            for (uint32_t k = 0; k < n; ++k) {
                const uint64_t fo = keep[idx[k]];
                uint32_t want = 0;
                pread(fd, &want, 4, (off_t) fo);
                (out[k] == want ? okc : bad)++;
            }
            std::printf("   GPU probe %s: %u reads, %d match pread, %d differ (requestResidency %.1f ms, run %.1f ms, status %d)\n", what, n,
                        okc, bad, treq, trun, (int) cb->status());
            for (uint32_t i = 0; i < NP; ++i) offs[i] = keep[i];
            r2->release();
        };
        if (whole) run_probe(whole, 0, "whole buffer");
        if (all && !chunks.empty()) run_probe(chunks.back(), (chunks.size() - 1) * chunk, "last chunk");
        // a streaming GPU read over one chunk, cold then warm (page cache)
        if (all && !chunks.empty()) {
            MTL::Buffer* b = chunks[chunks.size() / 2];
            MTL::Buffer* bs = dev->newBuffer(4, MTL::ResourceStorageModeShared);
            for (int rep = 0; rep < 2; ++rep) {
                *(uint32_t*) bs->contents() = 0;
                MTL::ResidencySet* r2 = dev->newResidencySet(rd, &e);
                r2->addAllocation(b); r2->addAllocation(bs); r2->commit();
                SumArgs a{b->gpuAddress(), bs->gpuAddress(), b->length() / 16};
                double tr = now_ms();
                MTL::CommandBuffer* cb = q->commandBuffer();
                cb->useResidencySet(r2);
                MTL::ComputeCommandEncoder* enc = cb->computeCommandEncoder();
                enc->setComputePipelineState(p_sum);
                enc->setBytes(&a, sizeof a, 0);
                enc->dispatchThreads(MTL::Size(40 * 1024, 1, 1), MTL::Size(256, 1, 1));
                enc->endEncoding();
                cb->commit();
                cb->waitUntilCompleted();
                const double dt = now_ms() - tr;
                std::printf("   GPU streaming read of a %.2f GiB chunk (%s): %.1f ms = %.1f GB/s\n", b->length() / 1073741824.0,
                            rep == 0 ? "first touch" : "again", dt, b->length() / dt / 1e6);
                r2->release();
            }
            bs->release();
        }
        for (auto* b : chunks) b->release();
        if (whole) whole->release();
        boffs->release(); bout->release();
        munmap(m, mlen);
    }
    close(fd);
    pool->release();
    return 0;
}