Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
11.7 KB
Raw Blame
// metal/src/engine/ds4_host.cpp - host side of the ported dsv41 kernels: compile the MSL once, wrap the engine's own buffers,
// dispatch.  First entry point: the Q8 activation quantiser (ds4_gguf.cu:201).
//
// The two runtimes meet on pointers: strata::gpu owns the engine's device memory (a device pointer is its MTLBuffer's
// gpuAddress) and exposes host_view(device_pointer); mtl::Context::wrap() adopts that address without copying and a
// Dispatch names it through Arg::B().  See RIGS/m1002.md for the derivation.
#include "mtl_rt.h"

#include <strata/gpu/runtime.hpp>



#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>

#include <strata/gpu/runtime.hpp>
#include <vector>

namespace strata::dsv41::metal_host {
mtl::Context& finish_cx();   // ds4_finish.cpp's Context: every kernel that orders with the chunk path's dispatches uses it
int shared_lib();   // from ds4_finish.cpp: the library the shared Context compiled


namespace {
mtl::Context* g_cx = nullptr;
int g_lib = -1;
bool g_tried = false;

bool ensure(mtl::Context& cx) {
    if (g_cx == &cx && g_lib >= 0) return true;
    if (g_tried && g_lib < 0) return false;
    g_tried = true;
    // the kernel lives in its own file and is self-contained (metal_stdlib + ggml_grids.h)
    g_lib = cx.compile({"../src/engine/msl/ds4_quant.metal"}, "", false);
    if (g_lib < 0) {
        std::fprintf(stderr, "dsv41 metal: compiling ds4_quant.metal failed\n");
        return false;
    }
    g_cx = &cx;
    return true;
}
}  // namespace

// q8(x, ldx, n) -> q (int8, n bytes) and d (half, n/32); ldx is the stride in values between blocks of 32
bool q8_quant(mtl::Context& cx, const float* x, int ldx, int n, void* q, void* d) {
    if (!ensure(cx)) return false;
    const size_t xb = (size_t) ldx * (size_t) (n / 32) * sizeof(float), qb = (size_t) n, db = (size_t) (n / 32) * 2;
    mtl::Buffer bx = cx.wrap(strata::gpu::host_view(x), xb);
    mtl::Buffer bq = cx.wrap(q, qb);
    mtl::Buffer bd = cx.wrap(d, db);
    mtl::Pipeline p;
    try {
        p = cx.pipeline(g_lib, "ds4_q8_quant");
    } catch (...) {
        return false;
    }
    mtl::Dispatch ds;
    ds.p = &p;
    ds.args = {mtl::Arg::B(bx), mtl::Arg::V((unsigned) ldx), mtl::Arg::B(bq), mtl::Arg::B(bd)};
    ds.grid[0] = (uint32_t) (n / 32);
    ds.grid[1] = ds.grid[2] = 1;
    ds.tg[0] = 32;
    ds.tg[1] = ds.tg[2] = 1;
    mtl::require_fit(p, 32, "ds4_q8_quant");
    cx.run({ds});
    return true;
}

}  // namespace strata::dsv41::metal_host

namespace strata::dsv41::metal_host {
namespace {
mtl::Context* g_init_cx = nullptr;
int g_lib_init = -1;

mtl::Context& shared_cx() {
    if (!g_init_cx) {
        g_init_cx = new mtl::Context();
        g_lib_init = g_init_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);
        if (g_lib_init < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");
    }
    return *g_init_cx;
}
}  // namespace

// ds4_kernels.cu:1477 - pre[i] = (i % 8) == 0 ? 1 : 0 for i < T*8
bool init_pre(float* pre, int T, void*) {
    // cpu path (T*8 floats of a fixed pattern): the GPU route had to wrap a whole page to reach 32 bytes, and that aliasing
    // buffer overlapped the pages holding h
    void* host = strata::gpu::host_view(pre);
    if (!host) { std::fprintf(stderr, "dsv41 metal: init_pre has no host view\n"); return false; }
    float* p = (float*) host;
    for (int t = 0; t < T; ++t)
        for (int i = 0; i < 8; ++i) p[t * 8 + i] = (i == 0) ? 1.0f : 0.0f;
    return true;
}


// ds4_kernels.cu:130 - the BF16 embedding gather, hc copies a token
bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
    strata::dsv41::metal_host::shared_cx();
    if (g_lib_init < 0) { std::fprintf(stderr, "dsv41 metal: embed: no library\n"); return false; }
    // host_view() returns a different address from the device pointer on this runtime, so wrapping it wrote into a mapping the
    // engine never reads.  On Apple silicon a device pointer is a valid CPU address for shared storage: wrap that instead.
    // DS4_HOSTVIEW=1 restores the old behaviour.
    const bool use_hv = std::getenv("DS4_HOSTVIEW") != nullptr;
    void* hrows = use_hv ? strata::gpu::host_view((void*) rows) : (void*) rows;
    void* hh = use_hv ? strata::gpu::host_view(h) : (void*) h;
    if (!hrows || !hh) { std::fprintf(stderr, "dsv41 metal: embed has no host view\n"); return false; }
    if (std::getenv("DS4_DBG_HV"))
        std::fprintf(stderr, "DBG hostview: rows %p -> %p   h %p -> %p\n",
                     (void*) rows, hrows, (void*) h, hh);
    const size_t page = 16384;
    auto wrap = [&](void* host, size_t bytes) {
        return strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
    };
    mtl::Buffer br = wrap(hrows, (size_t) T * dim * 2);
    mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4);
    if (!br.handle || !bh.handle) { std::fprintf(stderr, "dsv41 metal: embed wrap failed\n"); return false; }
    const size_t dro = (size_t) (((uintptr_t) hrows) - (((uintptr_t) hrows) & ~(uintptr_t) (page - 1)));
    const size_t dho = (size_t) (((uintptr_t) hh) - (((uintptr_t) hh) & ~(uintptr_t) (page - 1)));
    mtl::Pipeline pl;
    try { pl = strata::dsv41::metal_host::shared_cx().pipeline(g_lib_init, "ds4_embed"); } catch (...) { return false; }
    mtl::Dispatch d;
    d.p = &pl;
    d.args = {mtl::Arg::B(br, dro), mtl::Arg::B(bh, dho), mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};
    d.grid[0] = (uint32_t) ((dim + 255) / 256);
    d.grid[1] = (uint32_t) T;
    d.grid[2] = 1;
    d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
    strata::dsv41::metal_host::shared_cx().run({d});
    if (std::getenv("DS4_DBG_EMBWRITE")) {
        strata::dsv41::metal_host::shared_cx().flush_runs();
        const float* f0 = (const float*) hh;
        const uint16_t* r16 = (const uint16_t*) hrows;
        std::fprintf(stderr, "DBG embwrite: h=%p rows=%p h_off=%zu rows_off=%zu\n", hh, hrows, dho, dro);
        std::fprintf(stderr, "DBG embwrite: rows[0..3]=%04x %04x %04x %04x  h[0..3]=%g %g %g %g\n",
                     (unsigned) r16[0], (unsigned) r16[1], (unsigned) r16[2], (unsigned) r16[3],
                     f0[0], f0[1], f0[2], f0[3]);
    }
    return true;
}


// ds4_model.cu:2737 - C[t][m] = sum_k A[m][k] * B[t][k], F32 operands, alpha = 1, beta = 0
bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc) {
    finish_cx();
    const int lib = shared_lib();
    if (lib < 0) return false;
    void* ha = strata::gpu::host_view((void*) A);
    void* hb = strata::gpu::host_view((void*) B);
    void* hc = strata::gpu::host_view((void*) C);
    if (!ha || !hb || !hc) { std::fprintf(stderr, "dsv41 metal: sgemm_tn has no host view\n"); return false; }
    const size_t page = 16384;
    auto wrap = [&](void* host, size_t bytes) {
        return finish_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
    };
    auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
    mtl::Buffer ba = wrap(ha, (size_t) m * lda * 4);
    mtl::Buffer bb = wrap(hb, (size_t) nt * ldb * 4);
    mtl::Buffer bc = wrap(hc, (size_t) nt * ldc * 4);
    if (!ba.handle || !bb.handle || !bc.handle) return false;
    mtl::Pipeline pl;
    try { pl = finish_cx().pipeline(lib, "ds4_sgemm_tn"); } catch (...) { return false; }
    mtl::Dispatch d;
    d.p = &pl;
    d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bc, off(hc)),
              mtl::Arg::V((unsigned) m), mtl::Arg::V((unsigned) nt), mtl::Arg::V((unsigned) k),
              mtl::Arg::V((unsigned) lda), mtl::Arg::V((unsigned) ldb), mtl::Arg::V((unsigned) ldc)};
    const uint64_t total = (uint64_t) m * (uint64_t) nt;
    d.grid[0] = (uint32_t) ((total + 127) / 128);
    d.grid[1] = d.grid[2] = 1;
    d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;
    finish_cx().run({d});
    if (std::getenv("DS4_DBG_SG")) {
        finish_cx().flush_runs();
        const float* fa = (const float*) ha; const float* fb = (const float*) hb; const float* fc = (const float*) hc;
        std::fprintf(stderr, "sgemm m %d nt %d k %d lda %d ldb %d ldc %d A %g %g B %g %g C %g %g %g\n", m, nt, k, lda, ldb, ldc, fa[0], fa[1], fb[0], fb[1], fc[0], fc[1], fc[2]);
    }
    return true;
}


// embed through raw device addresses: the struct carries the gpuAddress of rows and h, which the runtime's residency set keeps
// resident, so the kernel writes the engine's own memory instead of a CPU mirror.
struct Ds4EmbedArgsHost {
    uint64_t rows;
    uint64_t h;
    uint32_t dim;
    uint32_t hc;
};

bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc) {
    // the engine uploads emb_rows on ITS stream; this kernel runs on mtl::Context queue.  Order the two before reading.
    strata::gpu::stream_synchronize(nullptr);
    if (std::getenv("DS4_DBG_ROWS")) {   // what is the engine actually handing us as the embedding rows?
        const uint16_t* rv = (const uint16_t*) strata::gpu::host_view((void*) rows);
        std::fprintf(stderr, "DBG rows_in: ptr=%p view=%p first=%04x %04x %04x %04x\n",
                     (const void*) rows, (const void*) rv,
                     rv ? (unsigned) rv[0] : 0u, rv ? (unsigned) rv[1] : 0u,
                     rv ? (unsigned) rv[2] : 0u, rv ? (unsigned) rv[3] : 0u);
    }
    strata::dsv41::metal_host::shared_cx();
    const int lib = shared_lib();
    if (lib < 0) return false;
    Ds4EmbedArgsHost a{};
    a.rows = (uint64_t) (uintptr_t) rows;
    a.h = (uint64_t) (uintptr_t) h;
    a.dim = (uint32_t) dim;
    a.hc = (uint32_t) hc;
    mtl::Pipeline pl;
    try { pl = strata::dsv41::metal_host::shared_cx().pipeline(lib, "ds4_embed_j"); } catch (...) { return false; }
    mtl::Dispatch d;
    d.p = &pl;
    d.args = {mtl::Arg::V(a)};
    d.grid[0] = (uint32_t) ((dim + 255) / 256);
    d.grid[1] = (uint32_t) T;
    d.grid[2] = 1;
    d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
    strata::dsv41::metal_host::shared_cx().run({d});
    strata::dsv41::metal_host::shared_cx().flush_runs();
    if (std::getenv("DS4_DBG_CMP")) {
        const float* hv = (const float*) strata::gpu::host_view(h);
        {
            const size_t nv = (size_t) T * hc * dim;
            size_t nf = 0, nz = 0;
            for (size_t i = 0; i < nv; ++i) { const float v = hv[i]; if (!(v == v)) ++nf; else if (v == 0.0f) ++nz; }
            std::fprintf(stderr, "DBG cmp: after embed n=%.0f nonfinite=%zu zeros=%zu\n", (double) nv, nf, nz);
        }
        std::fprintf(stderr, "DBG cmp: view %g %g %g | h=%g h+1=%g\n",
                     hv ? hv[0] : -1.0f, hv ? hv[1] : -1.0f, hv ? hv[2] : -1.0f,
                     hv ? hv[(size_t) dim * hc] : -1.0f, hv ? hv[(size_t) dim * hc + 1] : -1.0f);
    }
    if (std::getenv("DS4_DBG_ARGS")) {
        mtl::Buffer bp = strata::dsv41::metal_host::shared_cx().wrap(
            (void*) (((uintptr_t) strata::gpu::host_view(h)) & ~(uintptr_t) 16383), 16384);
        mtl::Pipeline pp = strata::dsv41::metal_host::shared_cx().pipeline(lib, "ds4_args_probe");
        mtl::Dispatch pd;
        pd.p = &pp;
        pd.args = {mtl::Arg::B(bp, ((uintptr_t) strata::gpu::host_view(h)) & 16383),
                   mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};
        pd.grid[0] = 1; pd.grid[1] = pd.grid[2] = 1; pd.tg[0] = 32; pd.tg[1] = pd.tg[2] = 1;
        strata::dsv41::metal_host::shared_cx().run({pd});
        strata::dsv41::metal_host::shared_cx().flush_runs();
        float got[3] = {0, 0, 0};
        { const float* hv2 = (const float*) strata::gpu::host_view(h); if (hv2) { got[0] = hv2[0]; got[1] = hv2[1]; } }
        std::fprintf(stderr, "DBG args: kernel sees dim=%g hc=%g\n", got[0], got[1]);
    }
    return true;
}

}  // namespace strata::dsv41::metal_host