Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek Metal port: keep the CUDA/HIP paths as in 0.2.7 (chunk-path hc mix through cuBLAS, async embedding upload, no extra h16 rows, no diagnostics on hot paths, type_on_chunk true off Metal)
Niko122156ab127c871b
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex fe29d2034..24a72234f 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1,4 +1,4 @@-// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.// src/ds4/ds4_gguf.cu - a GGUF's BF16 matrices and MXFP4 experts on the device (see ds4_gguf.cuh).#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal@@ -8,7 +8,9 @@#endif#include "ds4_iq.cuh"-#include "ds4_gguf_cpu.hpp"+#if defined(STRATA_METAL)+#include "ds4_gguf_cpu.hpp" // gg_row_to_bf16 (the DS4_DBG_IQ check)+#endif#include "ds4_numerics.cuh"#include <cuda_fp16.h>@@ -1086,8 +1088,11 @@ bool type_on_chunk(int type) {// ds4_mm.metal instantiates the tile GEMM for IQ2_XXS / IQ2_XS / IQ2_S only; IQ1_M (the 2.00 bpw pack's type) has// the matvec kernel but no tile one, so a chunked prompt would stop in "kernel launch with no Metal dispatch".if (type == iq::kIQ1_S || type == iq::kIQ4_NL) return false; // (IQ1_M: the tile kernels since DS4_MM_INST(29) and iq_bs)-#endifreturn type_on_device(type);+#else+ (void) type;+ return true; // CUDA and HIP: the chunk path takes whatever the layer holds (types with no GPU kernel stay on the CPU lane)+#endif}namespace {@@ -1168,7 +1173,7 @@ x3::Mat view(const x3::Mat& m, int c0, int nc) {bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {if (n <= 0 || n > x3::kMaxMvJobs) return false;#if defined(STRATA_METAL)- if (std::getenv("DS4_MV_LOG")) { // how the engine batches its matvec jobs (one call = one quantiser dispatch)+ if (ds4_dbg("DS4_MV_LOG")) { // how the engine batches its matvec jobs (one call = one quantiser dispatch)static std::atomic<unsigned long long> calls{0}, jobsum{0};static std::atomic<unsigned long long> hist[9];static std::atomic<int> seen{0};@@ -1264,7 +1269,7 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {quant_flush(QI, s);for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);#if defined(STRATA_METAL)- if (std::getenv("DS4_DBG_Q8") && nj[2] > 0) { // DBG: the Q8_0 matvec's first job against a host recompute+ if (ds4_dbg("DS4_DBG_Q8") && nj[2] > 0) { // DBG: the Q8_0 matvec's first job against a host recomputeds4_metal_flush();const BfJob& b = J[2].j[0];const uint8_t* w = (const uint8_t*) strata::gpu::host_view(b.w);@@ -1295,7 +1300,7 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {return false;}#if defined(STRATA_METAL)- if (std::getenv("DS4_DBG_IQ") && nij > 0) { // DBG: the low-bit matvec's first job against a host decode of its rows+ if (ds4_dbg("DS4_DBG_IQ") && nij > 0) { // DBG: the low-bit matvec's first job against a host decode of its rowsds4_metal_flush();for (int ji = 0; ji < std::min(nij, 2); ++ji) {const iq::MvJob& b = IJ[ji];diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex f6fdaccc8..4c71cdea6 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -2153,7 +2153,7 @@ void zc_stage(unsigned long long* plan, const int* idx, const unsigned long longvoid init_pre(float* pre, int T, cudaStream_t s) {#if defined(STRATA_METAL)- if (std::getenv("DS4_SKIP_INITPRE")) return;+ if (ds4_dbg("DS4_SKIP_INITPRE")) return;#endif#if defined(STRATA_METAL)if (strata::dsv41::metal_host::init_pre(pre, T, s)) return; // the ported MSL kernel@@ -2517,7 +2517,7 @@ void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T,}void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s) {#if defined(STRATA_METAL)- if (std::getenv("DS4_SKIP_HCMEAN")) return;+ if (ds4_dbg("DS4_SKIP_HCMEAN")) return;#endif#if defined(STRATA_METAL)// hc_mean_kernel on the Metal dispatch; the launch below is the fallbackdiff --git a/src/dsv41/ds4_launch.h b/src/dsv41/ds4_launch.hindex 433f833ae..553f523a0 100644--- a/src/dsv41/ds4_launch.h+++ b/src/dsv41/ds4_launch.h@@ -26,3 +26,12 @@ namespace strata::gpu {inline void* host_view(const void* p) { return const_cast<void*>(p); }}#endif++// ds4_dbg("NAME"): the Mac engine's opt-in diagnostics (DS4_DBG_*, DS4_CPU_LOG, DS4_TLOG, ...). On Metal it is an environment+// lookup; everywhere else it is the constant false, so the CUDA and HIP engines carry none of those checks on their hot paths.+#if defined(STRATA_METAL)+#include <cstdlib>+inline bool ds4_dbg(const char* name) { return std::getenv(name) != nullptr; }+#else+constexpr bool ds4_dbg(const char*) { return false; }+#endifdiff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex d576e348f..c43547498 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -24,6 +24,7 @@#include <unistd.h>#include <libproc.h> // proc_pid_rusage: the bytes this process had the storage read (io_read_bytes)#endif+#include "ds4_launch.h" // ds4_dbg#include "ds4_model.hpp"#include "ds4_experts_rule.hpp"#include "ds4_os.hpp"@@ -186,11 +187,11 @@ struct RoundTimes { // DS4_ROUND_TIMES: where a decode round's time goes (ms,unsigned long long draft_d = 0, verify_d = 0, step_d = 0;long rounds = 0, spec_rounds = 0, steps = 0;~RoundTimes() {- if (std::getenv("DS4_ROUND_TIMES") && rounds > 0)+ if (ds4_dbg("DS4_ROUND_TIMES") && rounds > 0)std::fprintf(stderr, "rounds: %ld (%ld speculative, %ld plain steps): draft %.1f ms (%.2f a round), verify %.1f ms (%.2f), plain step %.1f ms (%.2f)\n",rounds, spec_rounds, steps, draft, spec_rounds ? draft / (double) spec_rounds : 0.0, verify, spec_rounds ? verify / (double) spec_rounds : 0.0,step, steps ? step / (double) steps : 0.0);- if (std::getenv("DS4_ROUND_TIMES") && rounds > 0)+ if (ds4_dbg("DS4_ROUND_TIMES") && rounds > 0)std::fprintf(stderr, "dispatches a round: draft %.0f, verify %.0f, plain step %.0f\n", spec_rounds ? (double) draft_d / (double) spec_rounds : 0.0, spec_rounds ? (double) verify_d / (double) spec_rounds : 0.0, steps ? (double) step_d / (double) steps : 0.0);}} g_rt;@@ -1464,7 +1465,7 @@ int main(int argc, char** argv) {std::fprintf(stderr, "prompt: %zu tokens in %.0f ms (%.2f tok/s, windows of %d)\n", ids.size(), t1 - t0,1000.0 * (double) ids.size() / (t1 - t0), window);#if defined(STRATA_METAL)- if (std::getenv("DS4_SEQ_LOG")) std::fprintf(stderr, "dispatch seq after the prompt: %llu\n", ds4_dispatch_seq());+ if (ds4_dbg("DS4_SEQ_LOG")) std::fprintf(stderr, "dispatch seq after the prompt: %llu\n", ds4_dispatch_seq());#endifconst strata::ds4::Stats s0 = m.stats();if (s0.eg_rows > 0)@@ -1489,7 +1490,7 @@ int main(int argc, char** argv) {}std::printf("%d,", tok);std::fflush(stdout);- if (std::getenv("DS4_PROGRESS") && out.size() % 200 == 0) { // (sustained-run time series)+ if (ds4_dbg("DS4_PROGRESS") && out.size() % 200 == 0) { // (sustained-run time series)static double tp = now_ms();static size_t np = 0;const double tn = now_ms();diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex d5062736b..37d3537a0 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -103,6 +103,22 @@ int env_tri(const char* name) {double now_ms() {return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();}+// rows past the chunk that the Metal expert tile may read (its 16-row load of the last partial tile); CUDA and HIP allocate none+#if defined(STRATA_METAL)+constexpr size_t kH16Pad = 16;+#else+constexpr size_t kH16Pad = 0;+#endif+// the embedding rows' upload: asynchronous on the device's stream; on Metal a synchronous copy (the batched dispatches read the rows when+// they are committed, and h_emb is written again by the next window)+inline cudaError_t emb_upload(void* dst, const void* src, size_t n, cudaStream_t s) {+#if defined(STRATA_METAL)+ (void) s;+ return cudaMemcpy(dst, src, n, cudaMemcpyHostToDevice);+#else+ return cudaMemcpyAsync(dst, src, n, cudaMemcpyHostToDevice, s);+#endif+}size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }@@ -1265,7 +1281,7 @@ struct Model::Impl {return e;}void prof_mark(const Dev& d, int stage) {- if (std::getenv("DS4_TLOG")) { // the host's own phase times (what the GPU meters do not see)+ if (ds4_dbg("DS4_TLOG")) { // the host's own phase times (what the GPU meters do not see)static double t_last = 0, acc[5] = {0, 0, 0, 0, 0}; static long n = 0; static bool reg = false;if (!reg) {reg = true;@@ -1862,6 +1878,7 @@ struct Model::Impl {for (int r = 0; r < 3; ++r)if (!gg::type_on_device(l.lay.K[r])) {l.cpu_only = true;+#if defined(STRATA_METAL)static bool said = false; // (a mixed pack has many such layers: one line, not one a layer)if (!said) {said = true;@@ -1870,6 +1887,12 @@ struct Model::Impl {"experts stay on the CPU lane (ggml-cpu's rows; VRAM tier: none)\n",gg::type_name(l.lay.K[r]));}+#else+ std::fprintf(stderr,+ "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "+ "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",+ gg::type_name(l.lay.K[r]));+#endifbreak;}return true;@@ -2529,7 +2552,7 @@ struct Model::Impl {dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);- if (ok && std::getenv("DS4_DBG_DEVH"))+ if (ok && ds4_dbg("DS4_DBG_DEVH"))std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);if (!ok) return false;d.attn_scr.max_t = kAttnScratchT;@@ -2697,7 +2720,7 @@ struct Model::Impl {err = "embed.weight: read failed";return false;}- if (std::getenv("DS4_DBG_EMB")) std::fprintf(stderr,+ if (ds4_dbg("DS4_DBG_EMB")) std::fprintf(stderr,"DBG emb pinned: %04x %04x %04x %04x (row1: %04x %04x)\n",(unsigned) embed_pinned[0], (unsigned) embed_pinned[1],(unsigned) embed_pinned[2], (unsigned) embed_pinned[3],@@ -3244,7 +3267,7 @@ struct Model::Impl {dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&- dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&+ dalloc(d, c.h16, (C + kH16Pad) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&halloc(c.bounds_pin, 2048, err);if (!ok) return false;if (has_first && !dalloc(d, c.emb, C * D, err)) return false;@@ -6088,7 +6111,7 @@ struct Model::Impl {std::memcpy(d.second.first, r.dst, (size_t) hd);std::memcpy(d.second.second, r.scales, (size_t) hd / 32);}- if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite+ if (ds4_dbg("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finitefor (size_t i = 0; i < b.reads.size(); ++i) {const EgRead& r = b.reads[i];for (int k = 0; k < hd / 32; ++k) {@@ -6312,7 +6335,7 @@ struct Model::Impl {la_guess.assign(L.size(), {});pool = new LaPool();}- if (std::getenv("DS4_CPU_LOG") && !reg) {+ if (ds4_dbg("DS4_CPU_LOG") && !reg) {reg = true;static Impl* self = this;std::atexit([] { std::fprintf(stderr, "lookahead: %ld of %ld CPU-lane experts were guessed; %ld matrices queued\n", self->la_hit, self->la_all, self->la_issued); });@@ -6352,8 +6375,9 @@ struct Model::Impl {return;}static double t_pre = 0, t_q1 = 0, t_r1 = 0, t_q2 = 0, t_r2 = 0; static long n_calls = 0; static bool reg = false;- if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu_experts_iq phases (ms over %ld calls): prefetch %.1f, quant1 %.1f, gate_up %.1f, quant2 %.1f, down %.1f\n", n_calls, t_pre, t_q1, t_r1, t_q2, t_r2); }); }- ++n_calls;+ if (ds4_dbg("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu_experts_iq phases (ms over %ld calls): prefetch %.1f, quant1 %.1f, gate_up %.1f, quant2 %.1f, down %.1f\n", n_calls, t_pre, t_q1, t_r1, t_q2, t_r2); }); }+ const bool cpulog = ds4_dbg("DS4_CPU_LOG"); // (the Mac engine's counters; false elsewhere)+ if (cpulog) ++n_calls;double tp0 = now_ms();const float lim = g.lim;{ // (os::prefetch_cold, as above: the resident ones are left alone - on Windows the hint walks every page of a@@ -6365,7 +6389,7 @@ struct Model::Impl {for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});static const bool pf_all = std::getenv("DS4_IQ_PF_COLD") == nullptr || std::atoi(std::getenv("DS4_IQ_PF_COLD")) == 0;#if defined(__APPLE__) // (macOS: the cold reads of a call's matrices in parallel, and a page-cache census under DS4_CPU_LOG)- if (pf_all && std::getenv("DS4_CPU_LOG")) { // how many of the experts' matrices are in the page cache (mincore, the first 64 pages)+ if (pf_all && ds4_dbg("DS4_CPU_LOG")) { // how many of the experts' matrices are in the page cache (mincore, the first 64 pages)static long cold_n = 0, warm_n = 0; static double cold_ms = 0, warm_ms = 0; static bool reg2 = false;if (!reg2) { reg2 = true; std::atexit([] { std::fprintf(stderr, "prefetch: %ld matrices cold (%.1f ms), %ld warm (%.1f ms)\n", cold_n, cold_ms, warm_n, warm_ms); }); }for (const auto& w : want) {@@ -6393,7 +6417,7 @@ struct Model::Impl {else os::prefetch_cold(want);#endif}- double tp1 = now_ms(); t_pre += tp1 - tp0;+ double tp1 = now_ms(); if (cpulog) t_pre += tp1 - tp0;// the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of// entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or// chunk paths), then (x no longer needed) the slot's h@@ -6412,7 +6436,7 @@ struct Model::Impl {gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));}}- double tq1 = now_ms(); t_q1 += tq1 - tp1;+ double tq1 = now_ms(); if (cpulog) t_q1 += tq1 - tp1;std::function<void(int)> f[4];for (int X = 0; X < NH; ++X)f[X] = [&, X](int j) {@@ -6430,12 +6454,12 @@ struct Model::Impl {r0 - X * FFh, r1 - X * FFh, lim);};run_nodes(ne * GB, f);- double tr1 = now_ms(); t_r1 += tr1 - tq1;+ double tr1 = now_ms(); if (cpulog) t_r1 += tr1 - tq1;for (const Ent& en : ents)for (int i = 0; i < en.n; ++i)gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,aq + (size_t) (kWin + en.slot0 + i) * E * 4);- double tq2 = now_ms(); t_q2 += tq2 - tr1;+ double tq2 = now_ms(); if (cpulog) t_q2 += tq2 - tr1;for (int X = 0; X < NH; ++X)f[X] = [&, X](int j) {const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];@@ -6449,7 +6473,7 @@ struct Model::Impl {gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);};run_nodes(ne * DB, f);- t_r2 += now_ms() - tq2;+ if (cpulog) t_r2 += now_ms() - tq2;{ // DS4_PROF: where the CPU lane's time goes (act prep + prefetch, gate/up, h quant, down), summedstatic const bool on = std::getenv("DS4_PROF") != nullptr;if (on) {@@ -7408,16 +7432,16 @@ struct Model::Impl {Dev& d = devs[(size_t) d_dev];cudaSetDevice(d.id);for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));+ CK(emb_upload(d.emb_rows, h_emb, (size_t) T * D * 2, d.st));embed(d.emb_rows, T, d.h, D, H, d.st);- if (std::getenv("DS4_DBG_EMB")) {+ if (ds4_dbg("DS4_DBG_EMB")) {ds4_metal_flush();float *hp = d.h;float hh[4] = {0,0,0,0};cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);}- if (std::getenv("DS4_DBG_EMB")) {+ if (ds4_dbg("DS4_DBG_EMB")) {ds4_metal_flush();float hh[4] = {0, 0, 0, 0};cudaMemcpy(hh, d.h, sizeof(hh), cudaMemcpyDeviceToHost);@@ -7746,9 +7770,14 @@ struct Model::Impl {}prof_mark(d, 4);// attention+#if defined(STRATA_METAL)// the mix and the split in one dispatch (hc_mix + hc_split were two, back to back on the same token)hc_mix_split(c.h, l.hca_fn, g.hc * g.dim, (2 + g.hc) * g.hc, T, g.eps, c.mix, l.hca_scale, l.hca_base, H,g.sinkhorn, g.hc_eps, c.a_pre, c.a_post, c.a_comb, s);+#else+ if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;+ hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);+#endifdump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);@@ -7971,9 +8000,14 @@ struct Model::Impl {if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)prof_mark(d, 1);// the MoE+#if defined(STRATA_METAL)// the FFN half of the same fusion: mix and split in one dispatchhc_mix_split(c.h, l.hcf_fn, g.hc * g.dim, (2 + g.hc) * g.hc, T, g.eps, c.mix, l.hcf_scale, l.hcf_base, H,g.sinkhorn, g.hc_eps, c.f_pre, c.f_post, c.f_comb, s);+#else+ if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;+ hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);+#endifhc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)@@ -8254,7 +8288,7 @@ struct Model::Impl {CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));CK(cudaEventRecord(c.emb_ev[slot], s));embed(c.emb, T, c.h, D, H, s);- if (std::getenv("DS4_DBG_EMB")) {+ if (ds4_dbg("DS4_DBG_EMB")) {ds4_metal_flush();float *hp = c.h;float hh[4] = {0,0,0,0};@@ -8427,7 +8461,7 @@ struct Model::Impl {CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));CK(cudaEventRecord(c.emb_ev[slot], s));embed(c.emb, T, c.h, D, H, s);- if (std::getenv("DS4_DBG_EMB")) {+ if (ds4_dbg("DS4_DBG_EMB")) {ds4_metal_flush();float *hp = c.h;float hh[4] = {0,0,0,0};@@ -8937,7 +8971,7 @@ struct Model::Impl {: yoco_mode() == 2 ? g.win : (g.nl - yl) * (g.win - 1) + g.win;const int a = yl > 0 && n - keep >= kYocoMin ? n - keep : 0;const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;- if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);+ if (ds4_dbg("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);Run run;for (int x = 0; x < n; x += per) {run.p0.push_back(pos + x);@@ -9394,13 +9428,13 @@ struct Model::Impl {bool forward_serial(int64_t p0, int T, std::string& err) {const int D = g.dim, H = g.hc;int cur = L[0].dev;- if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();+ if (ds4_dbg("DS4_SYNC_WIN")) cudaDeviceSynchronize();{Dev& d = devs[(size_t) cur];cudaSetDevice(d.id);- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));+ CK(emb_upload(d.emb_rows, h_emb, (size_t) T * D * 2, d.st));embed(d.emb_rows, T, d.h, D, H, d.st);- if (std::getenv("DS4_DBG_EMB")) {+ if (ds4_dbg("DS4_DBG_EMB")) {ds4_metal_flush();float *hp = d.h;float hh[4] = {0,0,0,0};@@ -9408,7 +9442,7 @@ struct Model::Impl {std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);}init_pre(d.pre, T, d.st);- if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);+ if (ds4_dbg("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);}std::vector<float> dump;if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);@@ -9435,7 +9469,7 @@ struct Model::Impl {cur = l.dev;}if (!tap(devs[(size_t) l.dev], il, T, err)) return false;- if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);+ if (il == 0 && ds4_dbg("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);if (!layer(l, 0, p0, T, wplan, err)) return false;if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layersDev& dd = devs[(size_t) l.dev];@@ -9508,9 +9542,9 @@ struct Model::Impl {if (!started) { // the window's input: the embedding rows, or the previous group's hand-offstarted = true;if (il == 0) {- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));+ CK(emb_upload(d.emb_rows, h_emb, (size_t) T * D * 2, s));embed(d.emb_rows, T, d.h, D, H, s);- if (std::getenv("DS4_DBG_EMB")) {+ if (ds4_dbg("DS4_DBG_EMB")) {ds4_metal_flush();float *hp = d.h;float hh[4] = {0,0,0,0};@@ -9587,7 +9621,7 @@ struct Model::Impl {const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;const ArGroup& ag = groups[(size_t) L0.grp].ar;const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;- if (std::getenv("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file's+ if (ds4_dbg("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file'sds4_metal_flush();int shown = 0;for (int e = 0; e < l.n_exp && shown < 3; ++e) {@@ -9602,8 +9636,8 @@ struct Model::Impl {++shown;}}- if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);- if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG+ if (ds4_dbg("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);+ if (ds4_dbg("DS4_DBG_H") && mi == 0 && (l.il < 2 || ds4_dbg("DS4_DBG_ALL"))) { // DBGstd::vector<float> hv((size_t) T * H * D);cudaSetDevice(d.id);ds4_metal_flush();@@ -9641,7 +9675,7 @@ struct Model::Impl {engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);- if (std::getenv("DS4_DBG_EG")) {+ if (ds4_dbg("DS4_DBG_EG")) {dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);const float* xv = (const float*) strata::gpu::host_view(d.eg_x);for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)@@ -9656,13 +9690,13 @@ struct Model::Impl {}}if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;- if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);+ if (ds4_dbg("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);- if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);+ if (ds4_dbg("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);}dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));@@ -9960,7 +9994,7 @@ struct Model::Impl {if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);// DS4_TLOG2: the expert stage's three host spans (gate/up issue, the CPU-lane wait, down issue)- static const bool tlog2 = std::getenv("DS4_TLOG2") != nullptr;+ static const bool tlog2 = ds4_dbg("DS4_TLOG2");static double t_moe_gu = 0, t_moe_wait = 0, t_moe_dn = 0; static long t_moe_n = 0; static bool t_moe_reg = false;if (tlog2 && !t_moe_reg) {t_moe_reg = true;@@ -10035,7 +10069,7 @@ struct Model::Impl {return false;}// (glog rows are 1024 apart: route reads them so)- static const bool phases = std::getenv("DS4_PHASES") != nullptr; // debugging: where a decode layer's host time goes+ static const bool phases = ds4_dbg("DS4_PHASES"); // debugging: where a decode layer's host time goesstatic double ph[8] = {0}, ph_prev = 0; static long ph_n = 0;const double ph0 = phases ? now_ms() : 0.0;gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);@@ -10066,10 +10100,10 @@ struct Model::Impl {if (la_n > 0) CK(cudaEventRecord(d.ev2, s));}#endif- if (std::getenv("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);+ if (ds4_dbg("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);std::fprintf(stderr, "DBG s1 fmt=%d K=%d k=%d n=%d ld=%d raw=%p\n", l.s1.fmt, l.s1.K, l.s1.k, l.s1.n, l.s1.ld, (void*) l.s1.raw); }swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);- if (std::getenv("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);+ if (ds4_dbg("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;const double tw = now_ms();CK(cudaEventSynchronize(d.ev));@@ -10150,7 +10184,7 @@ struct Model::Impl {if (nrows > 0) {gather_rows(d.x, ga, nrows, D, d.xe, s);dump_sub(d, "xe", l.il, p0, nrows, d.xe, (size_t) nrows * D * 4);- { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }+ if (dump_s != nullptr) { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }dump_sub_host("vmap", l.il, p0, nvex, vmap.data(), vmap.size() * 4); }x3::MvJob j[2 * kWin * kPicks];int nj = 0;@@ -10178,9 +10212,9 @@ struct Model::Impl {ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)ds4_busy_begin(); // (and the GPU is kept out of its idle clocks until the lane is done)#endif- static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)+ static const bool skip_cpu = ds4_dbg("DS4_SKIP_CPU"); // (timing only: the CPU lane's experts left out)if (!skip_cpu) cpu_experts(l, h_x, ents);- else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)+ else if (const char* fk = ds4_dbg("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)#if defined(STRATA_METAL)ds4_busy_end();#endif@@ -10191,9 +10225,9 @@ struct Model::Impl {const float* y = c_yd.data() + (size_t) sl * D;for (int e = 0; e < D; ++e) o[e] += wv * y[e];}- { static double ms = 0; static long calls = 0, exps = 0; static bool reg = false;+ if (ds4_dbg("DS4_CPU_LOG")) { static double ms = 0; static long calls = 0, exps = 0; static bool reg = false;ms += now_ms() - tc; ++calls; exps += (long) ents.size();- if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu lane calls: %ld, experts %ld, %.1f ms (%.3f ms a call, %.3f ms an expert)%c", calls, exps, ms, ms / (double) calls, ms / (double) exps, 10); }); }+ if (ds4_dbg("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu lane calls: %ld, experts %ld, %.1f ms (%.3f ms a call, %.3f ms an expert)%c", calls, exps, ms, ms / (double) calls, ms / (double) exps, 10); }); }(void) 0; (void) ms; }st.cpu_ms += now_ms() - tc;dump_sub_host("cpu_part", l.il, p0, T, h_part, (size_t) T * D * 4);@@ -10201,7 +10235,7 @@ struct Model::Impl {}combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);dump_sub(d, "moe_out", l.il, p0, T, d.moe_out, (size_t) T * D * 4);- if (std::getenv("DS4_DBG_MOE")) {+ if (ds4_dbg("DS4_DBG_MOE")) {float hp = 0.0f;for (size_t i = 0; i < (size_t) T * D; ++i) hp = std::max(hp, std::fabs(h_part[i]));std::fprintf(stderr, "DBG moe il=%d T=%d vram_rows=%d cpu_slots=%d max|h_part|=%.4g", l.il, T, nrows, nslots, (double) hp);@@ -10323,7 +10357,7 @@ bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {Impl& p = *p_;p.pm.last = -1;p.halted = false;- if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);+ if (ds4_dbg("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path hereconst double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)