Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: probes that establish what is NOT wrong, and the honest state of the targets
Niko1221782a8381de35
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.hindex 1d5fa620c..d380dfd9c 100644--- a/metal/include/metal_compat/ds4_metal_host.h+++ b/metal/include/metal_compat/ds4_metal_host.h@@ -52,12 +52,13 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, inbool swiglu(const float* g, const float* u, int n, float lim, float* h);bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);-// the entry points, as metal/src/engine/ds4_finish.cpp defines them+// generated from ds4_host.cpp and ds4_finish.cppbool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out);bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out);bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);+bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc);bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps);diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex d14bf1748..a39ae9122 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -9,6 +9,7 @@#include "mtl_rt.h"#include <strata/gpu/runtime.hpp>+#include <cuda_runtime.h>#include <cstdio>#include <cstdint>@@ -44,12 +45,61 @@ int g_lib = -1;int g_lib_iq = -1;bool ensure() {+ if (std::getenv("DS4_DBG_MEM")) { // does a write through host_view read back through cudaMemcpy?+ void* p = nullptr;+ const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;+ if (cudaMalloc(&p, tsz) == cudaSuccess) {+ void* v = strata::gpu::host_view(p);+ if (v) {+ ((float*) v)[0] = 42.0f;+ float out = 0.0f;+ cudaMemcpy(&out, p, 4, cudaMemcpyDeviceToHost);+ std::fprintf(stderr, "DBG mem: gpu=%p view=%p wrote 42 through the view, cudaMemcpy gives %g\n", p, v, out);+ } else {+ std::fprintf(stderr, "DBG mem: gpu=%p has no host view\n", p);+ }+ cudaFree(p);+ }+ }if (g_cx) return g_lib >= 0;g_cx = new mtl::Context();g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);if (g_lib_iq < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_iq.metal failed\n");+ if (g_lib >= 0 && std::getenv("DS4_DBG_BIND")) { // write a constant by each route, copy it back+ void* p = nullptr;+ const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;+ if (cudaMalloc(&p, tsz) == cudaSuccess) {+ void* v = strata::gpu::host_view(p);+ float got[4] = {0, 0, 0, 0};+ {+ mtl::Buffer b = g_cx->wrap((void*) (((uintptr_t) v) & ~(uintptr_t) 16383), 16384);+ mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write");+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(b, ((uintptr_t) v) & 16383), mtl::Arg::V((unsigned) 4)};+ d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->flush_runs();+ cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);+ std::fprintf(stderr, "DBG bind A (wrap host_view + Arg::B): %g %g %g %g\n", got[0], got[1], got[2], got[3]);+ }+ {+ uint64_t pp = (uint64_t) (uintptr_t) p;+ mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write_v");+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::V(pp), mtl::Arg::V((unsigned) 4)};+ d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->flush_runs();+ cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);+ std::fprintf(stderr, "DBG bind B (device pointer by value): %g %g %g %g\n", got[0], got[1], got[2], got[3]);+ }+ cudaFree(p);+ }+ }return g_lib >= 0;}} // namespace@@ -1946,4 +1996,9 @@ mtl::Context& shared_cx() {return *g_cx;}+int shared_lib() {+ ensure();+ return g_lib;+}+} // namespace strata::dsv41::metal_host\ No newline at end of filediff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cppindex 9c0646d49..868da1444 100644--- a/metal/src/engine/ds4_host.cpp+++ b/metal/src/engine/ds4_host.cpp@@ -19,6 +19,8 @@#include <vector>namespace strata::dsv41::metal_host {+int shared_lib(); // from ds4_finish.cpp: the library the shared Context compiled+namespace {mtl::Context* g_cx = nullptr;@@ -85,7 +87,8 @@ mtl::Context& shared_cx() {// ds4_kernels.cu:1477 - pre[i] = (i % 8) == 0 ? 1 : 0 for i < T*8bool init_pre(float* pre, int T, void*) {strata::dsv41::metal_host::shared_cx();- if (g_lib_init < 0) return false;+ const int lib = shared_lib();+ if (lib < 0) return false;void* host = strata::gpu::host_view(pre);if (!host) { std::fprintf(stderr, "dsv41 metal: init_pre has no host view\n"); return false; }const size_t bytes = (size_t) T * 8 * sizeof(float);@@ -108,6 +111,7 @@ bool init_pre(float* pre, int T, void*) {d.tg[0] = 128;d.tg[1] = d.tg[2] = 1;strata::dsv41::metal_host::shared_cx().run({d});+ strata::dsv41::metal_host::shared_cx().flush_runs();return true;}@@ -116,9 +120,16 @@ bool init_pre(float* pre, int T, void*) {bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {strata::dsv41::metal_host::shared_cx();if (g_lib_init < 0) { std::fprintf(stderr, "dsv41 metal: embed: no library\n"); return false; }- void* hrows = strata::gpu::host_view((void*) rows);- void* hh = strata::gpu::host_view(h);+ // host_view() returns a different address from the device pointer on this runtime, so wrapping it wrote into a mapping the+ // engine never reads. On Apple silicon a device pointer is a valid CPU address for shared storage: wrap that instead.+ // DS4_HOSTVIEW=1 restores the old behaviour.+ const bool use_hv = std::getenv("DS4_HOSTVIEW") != nullptr;+ void* hrows = use_hv ? strata::gpu::host_view((void*) rows) : (void*) rows;+ void* hh = use_hv ? strata::gpu::host_view(h) : (void*) h;if (!hrows || !hh) { std::fprintf(stderr, "dsv41 metal: embed has no host view\n"); return false; }+ if (std::getenv("DS4_DBG_HV"))+ std::fprintf(stderr, "DBG hostview: rows %p -> %p h %p -> %p\n",+ (void*) rows, hrows, (void*) h, hh);const size_t page = 16384;auto wrap = [&](void* host, size_t bytes) {return strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);@@ -154,7 +165,8 @@ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {// ds4_model.cu:2737 - C[t][m] = sum_k A[m][k] * B[t][k], F32 operands, alpha = 1, beta = 0bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc) {strata::dsv41::metal_host::shared_cx();- if (g_lib_init < 0) return false;+ const int lib = shared_lib();+ if (lib < 0) return false;void* ha = strata::gpu::host_view((void*) A);void* hb = strata::gpu::host_view((void*) B);void* hc = strata::gpu::host_view((void*) C);@@ -183,4 +195,60 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, inreturn true;}-} // namespace strata::dsv41::metal_host++// embed through raw device addresses: the struct carries the gpuAddress of rows and h, which the runtime's residency set keeps+// resident, so the kernel writes the engine's own memory instead of a CPU mirror.+struct Ds4EmbedArgsHost {+ uint64_t rows;+ uint64_t h;+ uint32_t dim;+ uint32_t hc;+};++bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc) {+ // the engine uploads emb_rows on ITS stream; this kernel runs on mtl::Context queue. Order the two before reading.+ strata::gpu::stream_synchronize(nullptr);+ if (std::getenv("DS4_DBG_ROWS")) { // what is the engine actually handing us as the embedding rows?+ const uint16_t* rv = (const uint16_t*) strata::gpu::host_view((void*) rows);+ std::fprintf(stderr, "DBG rows_in: ptr=%p view=%p first=%04x %04x %04x %04x\n",+ (const void*) rows, (const void*) rv,+ rv ? (unsigned) rv[0] : 0u, rv ? (unsigned) rv[1] : 0u,+ rv ? (unsigned) rv[2] : 0u, rv ? (unsigned) rv[3] : 0u);+ }+ strata::dsv41::metal_host::shared_cx();+ const int lib = shared_lib();+ if (lib < 0) return false;+ Ds4EmbedArgsHost a{};+ a.rows = (uint64_t) (uintptr_t) rows;+ a.h = (uint64_t) (uintptr_t) h;+ a.dim = (uint32_t) dim;+ a.hc = (uint32_t) hc;+ mtl::Pipeline pl;+ try { pl = strata::dsv41::metal_host::shared_cx().pipeline(lib, "ds4_embed_j"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::V(a)};+ d.grid[0] = (uint32_t) ((dim + 255) / 256);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ strata::dsv41::metal_host::shared_cx().run({d});+ if (std::getenv("DS4_DBG_ARGS")) {+ mtl::Buffer bp = strata::dsv41::metal_host::shared_cx().wrap(+ (void*) (((uintptr_t) strata::gpu::host_view(h)) & ~(uintptr_t) 16383), 16384);+ mtl::Pipeline pp = strata::dsv41::metal_host::shared_cx().pipeline(lib, "ds4_args_probe");+ mtl::Dispatch pd;+ pd.p = &pp;+ pd.args = {mtl::Arg::B(bp, ((uintptr_t) strata::gpu::host_view(h)) & 16383),+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};+ pd.grid[0] = 1; pd.grid[1] = pd.grid[2] = 1; pd.tg[0] = 32; pd.tg[1] = pd.tg[2] = 1;+ strata::dsv41::metal_host::shared_cx().run({pd});+ strata::dsv41::metal_host::shared_cx().flush_runs();+ float got[3] = {0, 0, 0};+ { const float* hv2 = (const float*) strata::gpu::host_view(h); if (hv2) { got[0] = hv2[0]; got[1] = hv2[1]; } }+ std::fprintf(stderr, "DBG args: kernel sees dim=%g hc=%g\n", got[0], got[1]);+ }+ return true;+}++} // namespace strata::dsv41::metal_host\ No newline at end of filediff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex 710ece886..e07af11f7 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -1990,3 +1990,35 @@ kernel void ds4_confidence(constant Ds4ConfJob& j [[buffer(0)]], uint tg [[threaa = ds4_block_sum_n(a, red, lane, w, 32u);if (tid == 0u) j.out[t] = 1.0f / (1.0f + exp(-a));}++// embed by raw device addresses (the runtime's MTLResidencySet makes gpuAddress pointers valid kernel arguments)+struct Ds4EmbedArgs {+ device ushort* rows;+ device float* h;+ uint dim;+ uint hc;+};+kernel void ds4_embed_j(device const Ds4EmbedArgs* a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ ushort lane [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + uint(lane);+ if (e >= a->dim) return;+ const uint t = tgp.y;+ const uint bits = uint(a->rows[(size_t) t * a->dim + e]) << 16;+ const float v = as_type<float>(bits);+ for (uint s = 0; s < a->hc; ++s) a->h[((size_t) t * a->hc + s) * a->dim + e] = v;+}++// (diagnostic) write a constant through a device pointer passed as a value, and one passed in a buffer+kernel void ds4_test_write(device float* out [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {+ if (gid < n) out[gid] = 3.0f;+}+kernel void ds4_test_write_v(device const uint64_t* p [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {+ device float* out = (device float*) (uintptr_t) p[0];+ if (gid < n) out[gid] = 7.0f;+}++// (diagnostic) report the arguments the kernel received+kernel void ds4_args_probe(device float* out [[buffer(0)]], constant uint& dim [[buffer(1)]],+ constant uint& hc [[buffer(2)]], uint gid [[thread_position_in_grid]]) {+ if (gid == 0) { out[0] = float(dim); out[1] = float(hc); out[2] = 1.0f; }+}diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex c05b0f736..606c8555a 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1891,6 +1891,9 @@ void init_pre(float* pre, int T, cudaStream_t s) {}void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::embed_addr(rows, T, h, dim, hc)) return;+#endif#if defined(STRATA_METAL)if (strata::dsv41::metal_host::embed(rows, T, h, dim, hc, s)) return;#endifdiff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 63342a9f8..78f8c7f6b 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -2173,6 +2173,8 @@ struct Model::Impl {dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);+ if (ok && std::getenv("DS4_DBG_DEVH"))+ std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);if (!ok) return false;d.attn_scr.max_t = kAttnScratchT;d.attn_scr.cap = (int) topk_stride();