Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: probes that establish what is NOT wrong, and the honest state of the targets

Niko1221782a8381de35
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
index 1d5fa620c..d380dfd9c 100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -52,12 +52,13 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, in
bool swiglu(const float* g, const float* u, int n, float lim, float* h);
bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);
-// the entry points, as metal/src/engine/ds4_finish.cpp defines them
+// generated from ds4_host.cpp and ds4_finish.cpp
bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);
bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out);
bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);
bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out);
bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);
+bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc);
bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);
bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);
bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps);
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index d14bf1748..a39ae9122 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -9,6 +9,7 @@
#include "mtl_rt.h"
#include <strata/gpu/runtime.hpp>
+#include <cuda_runtime.h>
#include <cstdio>
#include <cstdint>
@@ -44,12 +45,61 @@ int g_lib = -1;
int g_lib_iq = -1;
bool ensure() {
+ if (std::getenv("DS4_DBG_MEM")) { // does a write through host_view read back through cudaMemcpy?
+ void* p = nullptr;
+ const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;
+ if (cudaMalloc(&p, tsz) == cudaSuccess) {
+ void* v = strata::gpu::host_view(p);
+ if (v) {
+ ((float*) v)[0] = 42.0f;
+ float out = 0.0f;
+ cudaMemcpy(&out, p, 4, cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG mem: gpu=%p view=%p wrote 42 through the view, cudaMemcpy gives %g\n", p, v, out);
+ } else {
+ std::fprintf(stderr, "DBG mem: gpu=%p has no host view\n", p);
+ }
+ cudaFree(p);
+ }
+ }
if (g_cx) return g_lib >= 0;
g_cx = new mtl::Context();
g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);
if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");
g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);
if (g_lib_iq < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_iq.metal failed\n");
+ if (g_lib >= 0 && std::getenv("DS4_DBG_BIND")) { // write a constant by each route, copy it back
+ void* p = nullptr;
+ const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;
+ if (cudaMalloc(&p, tsz) == cudaSuccess) {
+ void* v = strata::gpu::host_view(p);
+ float got[4] = {0, 0, 0, 0};
+ {
+ mtl::Buffer b = g_cx->wrap((void*) (((uintptr_t) v) & ~(uintptr_t) 16383), 16384);
+ mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write");
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::B(b, ((uintptr_t) v) & 16383), mtl::Arg::V((unsigned) 4)};
+ d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->flush_runs();
+ cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG bind A (wrap host_view + Arg::B): %g %g %g %g\n", got[0], got[1], got[2], got[3]);
+ }
+ {
+ uint64_t pp = (uint64_t) (uintptr_t) p;
+ mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write_v");
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::V(pp), mtl::Arg::V((unsigned) 4)};
+ d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;
+ g_cx->run({d});
+ g_cx->flush_runs();
+ cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);
+ std::fprintf(stderr, "DBG bind B (device pointer by value): %g %g %g %g\n", got[0], got[1], got[2], got[3]);
+ }
+ cudaFree(p);
+ }
+ }
return g_lib >= 0;
}
} // namespace
@@ -1946,4 +1996,9 @@ mtl::Context& shared_cx() {
return *g_cx;
}
+int shared_lib() {
+ ensure();
+ return g_lib;
+}
+
} // namespace strata::dsv41::metal_host
\ No newline at end of file
diff --git a/metal/src/engine/ds4_host.cpp b/metal/src/engine/ds4_host.cpp
index 9c0646d49..868da1444 100644
--- a/metal/src/engine/ds4_host.cpp
+++ b/metal/src/engine/ds4_host.cpp
@@ -19,6 +19,8 @@
#include <vector>
namespace strata::dsv41::metal_host {
+int shared_lib(); // from ds4_finish.cpp: the library the shared Context compiled
+
namespace {
mtl::Context* g_cx = nullptr;
@@ -85,7 +87,8 @@ mtl::Context& shared_cx() {
// ds4_kernels.cu:1477 - pre[i] = (i % 8) == 0 ? 1 : 0 for i < T*8
bool init_pre(float* pre, int T, void*) {
strata::dsv41::metal_host::shared_cx();
- if (g_lib_init < 0) return false;
+ const int lib = shared_lib();
+ if (lib < 0) return false;
void* host = strata::gpu::host_view(pre);
if (!host) { std::fprintf(stderr, "dsv41 metal: init_pre has no host view\n"); return false; }
const size_t bytes = (size_t) T * 8 * sizeof(float);
@@ -108,6 +111,7 @@ bool init_pre(float* pre, int T, void*) {
d.tg[0] = 128;
d.tg[1] = d.tg[2] = 1;
strata::dsv41::metal_host::shared_cx().run({d});
+ strata::dsv41::metal_host::shared_cx().flush_runs();
return true;
}
@@ -116,9 +120,16 @@ bool init_pre(float* pre, int T, void*) {
bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
strata::dsv41::metal_host::shared_cx();
if (g_lib_init < 0) { std::fprintf(stderr, "dsv41 metal: embed: no library\n"); return false; }
- void* hrows = strata::gpu::host_view((void*) rows);
- void* hh = strata::gpu::host_view(h);
+ // host_view() returns a different address from the device pointer on this runtime, so wrapping it wrote into a mapping the
+ // engine never reads. On Apple silicon a device pointer is a valid CPU address for shared storage: wrap that instead.
+ // DS4_HOSTVIEW=1 restores the old behaviour.
+ const bool use_hv = std::getenv("DS4_HOSTVIEW") != nullptr;
+ void* hrows = use_hv ? strata::gpu::host_view((void*) rows) : (void*) rows;
+ void* hh = use_hv ? strata::gpu::host_view(h) : (void*) h;
if (!hrows || !hh) { std::fprintf(stderr, "dsv41 metal: embed has no host view\n"); return false; }
+ if (std::getenv("DS4_DBG_HV"))
+ std::fprintf(stderr, "DBG hostview: rows %p -> %p h %p -> %p\n",
+ (void*) rows, hrows, (void*) h, hh);
const size_t page = 16384;
auto wrap = [&](void* host, size_t bytes) {
return strata::dsv41::metal_host::shared_cx().wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
@@ -154,7 +165,8 @@ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*) {
// ds4_model.cu:2737 - C[t][m] = sum_k A[m][k] * B[t][k], F32 operands, alpha = 1, beta = 0
bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc) {
strata::dsv41::metal_host::shared_cx();
- if (g_lib_init < 0) return false;
+ const int lib = shared_lib();
+ if (lib < 0) return false;
void* ha = strata::gpu::host_view((void*) A);
void* hb = strata::gpu::host_view((void*) B);
void* hc = strata::gpu::host_view((void*) C);
@@ -183,4 +195,60 @@ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, in
return true;
}
-} // namespace strata::dsv41::metal_host
+
+// embed through raw device addresses: the struct carries the gpuAddress of rows and h, which the runtime's residency set keeps
+// resident, so the kernel writes the engine's own memory instead of a CPU mirror.
+struct Ds4EmbedArgsHost {
+ uint64_t rows;
+ uint64_t h;
+ uint32_t dim;
+ uint32_t hc;
+};
+
+bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc) {
+ // the engine uploads emb_rows on ITS stream; this kernel runs on mtl::Context queue. Order the two before reading.
+ strata::gpu::stream_synchronize(nullptr);
+ if (std::getenv("DS4_DBG_ROWS")) { // what is the engine actually handing us as the embedding rows?
+ const uint16_t* rv = (const uint16_t*) strata::gpu::host_view((void*) rows);
+ std::fprintf(stderr, "DBG rows_in: ptr=%p view=%p first=%04x %04x %04x %04x\n",
+ (const void*) rows, (const void*) rv,
+ rv ? (unsigned) rv[0] : 0u, rv ? (unsigned) rv[1] : 0u,
+ rv ? (unsigned) rv[2] : 0u, rv ? (unsigned) rv[3] : 0u);
+ }
+ strata::dsv41::metal_host::shared_cx();
+ const int lib = shared_lib();
+ if (lib < 0) return false;
+ Ds4EmbedArgsHost a{};
+ a.rows = (uint64_t) (uintptr_t) rows;
+ a.h = (uint64_t) (uintptr_t) h;
+ a.dim = (uint32_t) dim;
+ a.hc = (uint32_t) hc;
+ mtl::Pipeline pl;
+ try { pl = strata::dsv41::metal_host::shared_cx().pipeline(lib, "ds4_embed_j"); } catch (...) { return false; }
+ mtl::Dispatch d;
+ d.p = &pl;
+ d.args = {mtl::Arg::V(a)};
+ d.grid[0] = (uint32_t) ((dim + 255) / 256);
+ d.grid[1] = (uint32_t) T;
+ d.grid[2] = 1;
+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;
+ strata::dsv41::metal_host::shared_cx().run({d});
+ if (std::getenv("DS4_DBG_ARGS")) {
+ mtl::Buffer bp = strata::dsv41::metal_host::shared_cx().wrap(
+ (void*) (((uintptr_t) strata::gpu::host_view(h)) & ~(uintptr_t) 16383), 16384);
+ mtl::Pipeline pp = strata::dsv41::metal_host::shared_cx().pipeline(lib, "ds4_args_probe");
+ mtl::Dispatch pd;
+ pd.p = &pp;
+ pd.args = {mtl::Arg::B(bp, ((uintptr_t) strata::gpu::host_view(h)) & 16383),
+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};
+ pd.grid[0] = 1; pd.grid[1] = pd.grid[2] = 1; pd.tg[0] = 32; pd.tg[1] = pd.tg[2] = 1;
+ strata::dsv41::metal_host::shared_cx().run({pd});
+ strata::dsv41::metal_host::shared_cx().flush_runs();
+ float got[3] = {0, 0, 0};
+ { const float* hv2 = (const float*) strata::gpu::host_view(h); if (hv2) { got[0] = hv2[0]; got[1] = hv2[1]; } }
+ std::fprintf(stderr, "DBG args: kernel sees dim=%g hc=%g\n", got[0], got[1]);
+ }
+ return true;
+}
+
+} // namespace strata::dsv41::metal_host
\ No newline at end of file
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index 710ece886..e07af11f7 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -1990,3 +1990,35 @@ kernel void ds4_confidence(constant Ds4ConfJob& j [[buffer(0)]], uint tg [[threa
a = ds4_block_sum_n(a, red, lane, w, 32u);
if (tid == 0u) j.out[t] = 1.0f / (1.0f + exp(-a));
}
+
+// embed by raw device addresses (the runtime's MTLResidencySet makes gpuAddress pointers valid kernel arguments)
+struct Ds4EmbedArgs {
+ device ushort* rows;
+ device float* h;
+ uint dim;
+ uint hc;
+};
+kernel void ds4_embed_j(device const Ds4EmbedArgs* a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ ushort lane [[thread_index_in_threadgroup]]) {
+ const uint e = tgp.x * 256u + uint(lane);
+ if (e >= a->dim) return;
+ const uint t = tgp.y;
+ const uint bits = uint(a->rows[(size_t) t * a->dim + e]) << 16;
+ const float v = as_type<float>(bits);
+ for (uint s = 0; s < a->hc; ++s) a->h[((size_t) t * a->hc + s) * a->dim + e] = v;
+}
+
+// (diagnostic) write a constant through a device pointer passed as a value, and one passed in a buffer
+kernel void ds4_test_write(device float* out [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {
+ if (gid < n) out[gid] = 3.0f;
+}
+kernel void ds4_test_write_v(device const uint64_t* p [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {
+ device float* out = (device float*) (uintptr_t) p[0];
+ if (gid < n) out[gid] = 7.0f;
+}
+
+// (diagnostic) report the arguments the kernel received
+kernel void ds4_args_probe(device float* out [[buffer(0)]], constant uint& dim [[buffer(1)]],
+ constant uint& hc [[buffer(2)]], uint gid [[thread_position_in_grid]]) {
+ if (gid == 0) { out[0] = float(dim); out[1] = float(hc); out[2] = 1.0f; }
+}
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index c05b0f736..606c8555a 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1891,6 +1891,9 @@ void init_pre(float* pre, int T, cudaStream_t s) {
}
void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::embed_addr(rows, T, h, dim, hc)) return;
+#endif
#if defined(STRATA_METAL)
if (strata::dsv41::metal_host::embed(rows, T, h, dim, hc, s)) return;
#endif
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 63342a9f8..78f8c7f6b 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -2173,6 +2173,8 @@ struct Model::Impl {
dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&
dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&
dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);
+ if (ok && std::getenv("DS4_DBG_DEVH"))
+ std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);
if (!ok) return false;
d.attn_scr.max_t = kAttnScratchT;
d.attn_scr.cap = (int) topk_stride();