Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Merge f-dsv41-mac (DeepSeek on Metal) into v0.2.7
Niko12218477811197d9
diff --cc include/strata/dsv41/exl3.hppindex a5455b75a,9ab2482fd..6fb409e62--- a/include/strata/dsv41/exl3.hpp+++ b/include/strata/dsv41/exl3.hpp@@@ -152,12 -152,10 +152,14 @@@ struct SetArgsconst int* bounds = nullptr;int row0 = 0;int max_rows = 0; // the most rows one expert of the set has+ const int* hbounds = nullptr; // (Metal) the same bounds on the host: [n_exp + 1], absolute rows+ int total_rows = 0; // (Metal) the set's rows in allint n_embd = 4096, n_ff = 2048;float limit = 0.0f;+ // gate / up: the caller's F16 copy of the activations (row_idx picks the rows). The tiles round F32 to F16 for+ // their shared-memory tile anyway, so the same values come out; this only saves the F32 read of the same rows by+ // every column block of an expert (18-36 reads of 1.8-3.7 MB on a prompt) - MarLin's "small operand cheap".+ const uint16_t* x16 = nullptr;};/// h[local row] = FP16 clamped swiglu of gate and up over x[row_idx[local row]] (F32 rows, ldx floats apart).void set_gate_up(const SetArgs& a, const float* x, int ldx, const int* row_idx, uint16_t* h16, int ldh, cudaStream_t s);diff --cc metal/include/metal_compat/ds4_metal_host.hindex 000000000,ac298c9db..9eb7c9a5dmode 000000,100644..100644--- a/metal/include/metal_compat/ds4_metal_host.h+++ b/metal/include/metal_compat/ds4_metal_host.h@@@ -1,0 -1,109 +1,111 @@@+ // metal/include/metal_compat/ds4_metal_host.h - the dsv41 kernels that have a Metal dispatch.+ //+ // GENERATED from metal/src/engine/ds4_host.cpp and ds4_finish.cpp, which are the source of truth: do not hand-edit the+ // declaration block, regenerate it (see RIGS/m1002.md).+ #pragma once++ #include <cstddef>+ #include <cstdint>+ #include <vector>++ namespace strata::gpu {+ struct StreamImpl;+ using Stream = StreamImpl*;+ } // namespace strata::gpu++ namespace strata::dsv41::metal_host {++ // ds4_kernels.cu:1894 finish_kernel<false> - the engine's Finish, as ds4_kernels.cuh declares it+ struct FinishIn {+ const float* w;+ float eps;+ int rd, pos, inverse, vpp, pos_step;+ const float* freqs;+ int quant;+ const uint16_t* dst;+ long long dst_stride;+ uint8_t* pdst;+ long long pdst_stride;+ };++ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);+ bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);+ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);+ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);+ bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);+ bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);+ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);+ bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);+ bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out);+ bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x);+ bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T, float* pre, float* post, float* comb);++bool hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post, float* comb);+ bool init_pre(float* pre, int T, void*);+ bool iq_mv(const void* jobs, int nj);+ bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);+ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);+ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);+ bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);+ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);+ bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);+ bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);+ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);+ bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out);+ uint64_t zc_map(const void* p, size_t bytes); // zero-copy weights: the GPU address of host memory, 0 if not mappable+ bool q8_quant_multi(const void* specs, int n); // {const float* x; uint64 ldx; uint32 rows, k; int8* q; float* d} each+ bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td);+ bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff, float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td);+ bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out);+ bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha, float beta);+ bool swiglu(const float* g, const float* u, int n, float lim, float* h);+ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);++ // generated from ds4_host.cpp and ds4_finish.cpp+ bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride, int nb_max);+ bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out);+ bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T, int d, const float* w, float eps, float* latent);+ bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out);+ bool embed(const uint16_t* rows, int T, float* h, int dim, int hc, void*);+ bool embed_addr(const uint16_t* rows, int T, float* h, int dim, int hc);+ bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt);+ bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt);+ bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps);+ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup);+ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);+ bool gather_rows(const float* x, const void* g, int n, int cols, float* out);+ bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy);+ bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy);+ bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride);+ bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);+ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);+ bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);+ bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out);+ bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x);+ bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T, float* pre, float* post, float* comb);++bool hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post, float* comb);+ bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand, int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt);+ bool init_pre(float* pre, int T, void*);+ bool iota_rows(int* out, int T, int n, int stride);+ bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq, const float* hd, float* part, int type, int groups);+ bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit, const int8_t* xq, const float* xd, float* h, int type, int groups);+ bool iq_mv(const void* jobs, int nj);+ bool markov_add(const float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R);+ bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim);+ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);+ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k);+ bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride, const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd);+ bool mx_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq, const float* hd, float* part, int groups);+ bool mx_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start, const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit, const int8_t* xq, const float* xd, float* h, int groups);+ bool mx_mv(const void* jobs, int nj, int max_n, size_t job_bytes, int fmt);+ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d);+ bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes);+ bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w, const float* bias_vl, const void* vl);+ bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram, int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl);+ bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n, float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out);+ bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);+ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows, const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt);+ bool swiglu(const float* g, const float* u, int n, float lim, float* h);+ bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out, int out_stride);+ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap, const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes);++ } // namespace strata::dsv41::metal_hostdiff --cc metal/kernels/moe_mm.metalindex fb2f6f056,ff28897ac..5207a5f2d--- a/metal/kernels/moe_mm.metal+++ b/metal/kernels/moe_mm.metal@@@ -635,12 -633,12 +635,13 @@@ template <> struct W8<T_IQ2_XXS>device const uchar* bl = row + (k >> 8) * 66;const int ib = (k & 255) >> 5, l = (k & 31) >> 3, hh = (k & 7) >> 2;const uint a0 = rd32(bl + 2 + 8 * ib), a1 = rd32(bl + 2 + 8 * ib + 4);-- threadgroup const uchar* ks = (threadgroup const uchar*)(tab + 512);const float sc = rdh(bl) * (0.5f + float(a1 >> 28)) * 0.25f;const uint i0 = (a0 >> (8 * l)) & 255, i1 = (a0 >> (8 * (l + 2))) & 255;-- const uint s0 = ks[(a1 >> (7 * l)) & 127], s1 = ks[(a1 >> (7 * (l + 2))) & 127];-- a = w4_signed(tab[2 * i0 + hh], hh ? s0 >> 4 : s0, sc);-- b = w4_signed(tab[2 * i1 + hh], hh ? s1 >> 4 : s1, sc);++ // through the accessors, so QUANT_CTAB (the constant grids, no threadgroup copy) gives the same values: the++ // direct tab[] reads below skipped the ctab branch and read an unfilled table++ const uint s0 = ksg(tab, 512, (a1 >> (7 * l)) & 127), s1 = ksg(tab, 512, (a1 >> (7 * (l + 2))) & 127);++ a = w4_signed(g_iq2xxs(tab, i0, hh), hh ? s0 >> 4 : s0, sc);++ b = w4_signed(g_iq2xxs(tab, i1, hh), hh ? s1 >> 4 : s1, sc);}};template <> struct W8<T_IQ2_XS> {@@@ -649,14 -647,14 +650,14 @@@const int ib = (k & 255) >> 5, l = (k & 31) >> 3, hh = (k & 7) >> 2;device const ushort* q = (device const ushort*)(bl + 2) + 4 * ib;const uint v0 = q[l], v1 = q[l + 2];-- threadgroup const uchar* ks = (threadgroup const uchar*)(tab + 1024);const uint scb = bl[66 + ib];const float d = rdh(bl);const float sc0 = d * (0.5f + float((scb >> (4 * (l >> 1))) & 15)) * 0.25f;const float sc1 = d * (0.5f + float((scb >> (4 * ((l + 2) >> 1))) & 15)) * 0.25f;-- const uint s0 = ks[v0 >> 9], s1 = ks[v1 >> 9];-- a = w4_signed(tab[2 * (v0 & 511) + hh], hh ? s0 >> 4 : s0, sc0);-- b = w4_signed(tab[2 * (v1 & 511) + hh], hh ? s1 >> 4 : s1, sc1);++ // through the accessors: the direct tab[]/ks[] reads ignored QUANT_CTAB, whose tables are then never filled++ const uint s0 = ksg(tab, 1024, v0 >> 9), s1 = ksg(tab, 1024, v1 >> 9);++ a = w4_signed(g_iq2xs(tab, v0 & 511, hh), hh ? s0 >> 4 : s0, sc0);++ b = w4_signed(g_iq2xs(tab, v1 & 511, hh), hh ? s1 >> 4 : s1, sc1);}};template <> struct W8<T_IQ2_S> {@@@ -675,6 -673,6 +676,42 @@@b = w4_signed(g_iq2s(tab, i1, hh), hh ? s1 >> 4 : s1, sc1);}};++++// IQ1_M (the 2.00 bpw pack's 1.75-bit type), ggml's dequantize_row_iq1_m: the block's F16 scale sits in the top++// nibbles of its four u16 scales; a 32-value sub-block has two 3-bit scales (groups 0-1 and 2-3, eight values each),++// and each eight-value group carries a +-0.125 delta picked by a bit of its qh byte. The tile's two halves are the++// four values at k and the four at k + 16, each decoded on its own (k + 16 crosses into the next sub-block at l >= 2).++template <typename P>++static inline half4 iq1m_w4(P row, int k) {++ const P bl = row + (k >> 8) * 56;++ const int ib = (k & 255) >> 5, g = (k & 31) >> 3, hh = (k & 7) >> 2;++ const uint s0 = uint(bl[48]) | (uint(bl[49]) << 8), s1 = uint(bl[50]) | (uint(bl[51]) << 8);++ const uint s2 = uint(bl[52]) | (uint(bl[53]) << 8), s3 = uint(bl[54]) | (uint(bl[55]) << 8);++ const float d = float(as_type<half>(ushort((s0 >> 12) | ((s1 >> 8) & 0x00f0u) | ((s2 >> 4) & 0x0f00u) | (s3 & 0xf000u))));++ const uint sc = uint(((const device ushort*) (bl + 48))[ib >> 1]);++ const float dl = d * float(2u * ((sc >> (6u * uint(ib & 1) + (g < 2 ? 0u : 3u))) & 7u) + 1u);++ const uint qh = (g < 2) ? uint(bl[32 + 2 * ib]) : uint(bl[33 + 2 * ib]);++ const uint idx = uint(bl[4 * ib + g]) | (((g & 1) != 0) ? ((qh << 4u) & 0x700u) : ((qh << 8u) & 0x700u));++ const bool neg = ((g & 1) != 0) ? ((qh & 0x80u) != 0u) : ((qh & 0x08u) != 0u);++ const ulong gr = iq1s_grid[idx];++ const float delta = neg ? -0.125f : 0.125f;++ float v[4];++ for (int j = 0; j < 4; ++j) {++ const uint by = uint((gr >> (8u * uint(4 * hh + j))) & 0xffu);++ const int c = (by & 0x80u) != 0u ? int(by) - 256 : int(by);++ v[j] = dl * (float(c) + delta);++ }++ return half4(v[0], v[1], v[2], v[3]);++}++template <> struct W8<T_IQ1_M> {++ template <typename P>++ static inline void run(P row, int k, threadgroup const uint* tab, thread half4& a, thread half4& b) {++ (void) tab;++ a = iq1m_w4(row, k);++ b = iq1m_w4(row, k + 16);++ }++};++template <> struct W8<T_IQ3_S> {template <typename P>static inline void run(P row, int k, threadgroup const uint* tab, thread half4& a, thread half4& b) {diff --cc metal/kernels/quant.metalindex 14df14eef,51fecca7c..cbed727cc--- a/metal/kernels/quant.metal+++ b/metal/kernels/quant.metal@@@ -25,6 -25,6 +25,7 @@@#define T_IQ3_XXS 18#define T_IQ4_NL 20#define T_IQ3_S 21++#define T_IQ1_M 29 // the 2.00 bpw pack's 1.75-bit type#define T_IQ2_S 22#define T_IQ4_XS 23#define T_Q2_0 42@@@ -39,6 -36,6 +40,7 @@@ template <> struct fmt<T_Q2_0> { enutemplate <> struct fmt<T_Q2_0R> { enum { QK = 64, BB = 18, UNIT = 16, TAB = 1 }; };template <> struct fmt<T_IQ3_XXS> { enum { QK = 256, BB = 98, UNIT = 16, TAB = 256 + 32 }; };template <> struct fmt<T_IQ2_XXS> { enum { QK = 256, BB = 66, UNIT = 16, TAB = 512 + 32 }; };++template <> struct fmt<T_IQ1_M> { enum { QK = 256, BB = 56, UNIT = 16, TAB = 1 }; };template <> struct fmt<T_IQ2_XS> { enum { QK = 256, BB = 74, UNIT = 16, TAB = 1024 + 32 }; };template <> struct fmt<T_IQ2_S> { enum { QK = 256, BB = 82, UNIT = 16, TAB = 2048 + 32 }; };template <> struct fmt<T_IQ3_S> { enum { QK = 256, BB = 110, UNIT = 16, TAB = 512 + 32 }; };@@@ -389,6 -112,6 +391,8 @@@ template <> inline void load_tab<T_IQ2_for (int i = tid; i < 512; i += nth) { tab[2 * i] = uint(iq2xs_grid[i]); tab[2 * i + 1] = uint(iq2xs_grid[i] >> 32); }load_signs(tab, 1024, tid, nth);}++// IQ1_M: no threadgroup table - the tile reads iq1s_grid from the constants (the w4 helper in moe_mm.metal)++template <> inline void load_tab<T_IQ1_M>(threadgroup uint*, ushort, ushort) {}template <> inline void load_tab<T_IQ2_S>(threadgroup uint* tab, ushort tid, ushort nth) {if (ctab) return;for (int i = tid; i < 1024; i += nth) { tab[2 * i] = uint(iq2s_grid[i]); tab[2 * i + 1] = uint(iq2s_grid[i] >> 32); }diff --cc metal/src/engine/ds4_finish.cppindex 000000000,c773eeed5..0bc8f4ad7mode 000000,100644..100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@@ -1,0 -1,2599 +1,2730 @@@+ // metal/src/engine/ds4_finish.cpp - the host side of ds4_finish (ds4_kernels.cu:1894, finish_kernel<false>).+ //+ // The kernel takes the engine's own device pointer for x as a bound buffer plus an offset, and the Ds4Finish struct as a constant+ // buffer whose pointer fields are the engine's gpuAddresses (Metal lets a kernel dereference them - ss the repo's note that+ // "kernels may take raw gpuAddress pointers"). Strides and counts travel as plain values.+ //+ // The mrope (rd > 0) and rounding (quant != 0) branches are not written yet, and neither is the KV pack (pdst): this function+ // returns false for those, so the engine reports the launch line instead of a step being skipped.+ #include "mtl_rt.h"++ #include <strata/gpu/runtime.hpp>+ #include <cuda_runtime.h>++ #include <cstdio>+ #include <cstdint>+ #include <cstring>+ #include <algorithm>+ #include <map>+ #include <vector>++ // called by the gpu layer's synchronisation: everything batched must reach the GPU before a host read+ extern "C" void ds4_metal_flush();+ extern "C" size_t ds4_gpu_alloc_remaining(const void* cpu);+ extern "C" void* ds4_gpu_buffer_for_cpu(const void* cpu, size_t bytes, size_t* off);+ extern "C" void ds4_gpu_sync_all_streams();+ extern "C" unsigned long long ds4_dispatch_seq();+ static unsigned long long g_q8_deduped = 0;++ #define DSV41_PROBE(fmt, ...) do { static bool once_ = false; if (!once_) { once_ = true; std::fprintf(stderr, "[probe] " fmt, __VA_ARGS__); } } while (0)++ namespace strata::dsv41::metal_host {++ // mirrors metal/src/engine/msl/ds4_init.metal's Ds4Finish (Metal has no 'long long': 64-bit is 'long')+ struct GpuFinish {+ uint64_t w; // device float* (the engine's gpuAddress, 0 = none)+ float eps;+ int rd, pos, inverse, vpp, pos_step;+ uint64_t freqs; // device float*+ int quant;+ uint64_t dst; // device ushort*+ int64_t dst_stride;+ uint64_t pdst; // device uchar*+ int64_t pdst_stride;+ };++ namespace {+ mtl::Context* g_cx = nullptr;+ int g_lib = -1;+ int g_lib_iq = -1;++ bool ensure() {+ if (std::getenv("DS4_DBG_MEM")) { // does a write through host_view read back through cudaMemcpy?+ void* p = nullptr;+ const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;+ if (cudaMalloc(&p, tsz) == cudaSuccess) {+ void* v = strata::gpu::host_view(p);+ if (v) {+ ((float*) v)[0] = 42.0f;+ float out = 0.0f;+ cudaMemcpy(&out, p, 4, cudaMemcpyDeviceToHost);+ std::fprintf(stderr, "DBG mem: gpu=%p view=%p wrote 42 through the view, cudaMemcpy gives %g\n", p, v, out);+ } else {+ std::fprintf(stderr, "DBG mem: gpu=%p has no host view\n", p);+ }+ cudaFree(p);+ }+ }+ if (g_cx) return g_lib >= 0;+ g_cx = new mtl::Context();+ g_cx->start_keepalive();+ g_lib = g_cx->compile({"../src/engine/msl/ds4_init.metal"}, "", false);+ if (g_lib < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_init.metal failed\n");+ g_lib_iq = g_cx->compile({"../src/engine/msl/ds4_iq.metal"}, "", false);+ if (g_lib_iq < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_iq.metal failed\n");+ if (g_lib >= 0 && std::getenv("DS4_DBG_BIND")) { // write a constant by each route, copy it back+ void* p = nullptr;+ const size_t tsz = std::getenv("DS4_MEM_GB") ? 64u << 20 : 4096u;+ if (cudaMalloc(&p, tsz) == cudaSuccess) {+ void* v = strata::gpu::host_view(p);+ float got[4] = {0, 0, 0, 0};+ {+ mtl::Buffer b = g_cx->wrap((void*) (((uintptr_t) v) & ~(uintptr_t) 16383), 16384);+ mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write");+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(b, ((uintptr_t) v) & 16383), mtl::Arg::V((unsigned) 4)};+ d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->flush_runs();+ cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);+ std::fprintf(stderr, "DBG bind A (wrap host_view + Arg::B): %g %g %g %g\n", got[0], got[1], got[2], got[3]);+ }+ {+ uint64_t pp = (uint64_t) (uintptr_t) p;+ mtl::Pipeline pl = g_cx->pipeline(g_lib, "ds4_test_write_v");+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::V(pp), mtl::Arg::V((unsigned) 4)};+ d.grid[0] = 1; d.grid[1] = d.grid[2] = 1; d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->flush_runs();+ cudaMemcpy(got, p, sizeof(got), cudaMemcpyDeviceToHost);+ std::fprintf(stderr, "DBG bind B (device pointer by value): %g %g %g %g\n", got[0], got[1], got[2], got[3]);+ }+ cudaFree(p);+ }+ }+ return g_lib >= 0;+ }+ } // namespace++ // the engine's Finish, as ds4_kernels.cuh declares it+ struct FinishIn {+ const float* w;+ float eps;+ int rd, pos, inverse, vpp, pos_step;+ const float* freqs;+ int quant;+ const uint16_t* dst;+ long long dst_stride;+ uint8_t* pdst;+ long long pdst_stride;+ };++ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {+ if (n_vec <= 0 || len <= 0) return true;+ // finish_kernel<true> (the KV pack: the row quantised to codes + group scales) or <false>; the <false> rounding+ // branch (quant without pdst) is the one still unwritten+ const bool pack = f.pdst != nullptr;+ if (!ensure()) return false;+ void* hx = strata::gpu::host_view(x);+ if (!hx) { std::fprintf(stderr, "dsv41 metal: finish has no host view\n"); return false; }+ const size_t page = 16384;+ const size_t bytes = (size_t) (n_vec - 1) * (size_t) stride * sizeof(float) + (size_t) len * sizeof(float);+ const size_t base = ((uintptr_t) hx) & ~(uintptr_t) (page - 1);+ const size_t off = ((uintptr_t) hx) - base;+ mtl::Buffer bx = g_cx->wrap((void*) base, ((bytes + off + page - 1) / page) * page);+ if (!bx.handle) { std::fprintf(stderr, "dsv41 metal: finish wrap failed\n"); return false; }++ GpuFinish gf{};+ gf.w = (uint64_t) (uintptr_t) f.w;+ gf.eps = f.eps;+ gf.rd = f.rd; gf.pos = f.pos; gf.inverse = f.inverse; gf.vpp = f.vpp; gf.pos_step = f.pos_step;+ gf.freqs = (uint64_t) (uintptr_t) f.freqs;+ gf.quant = f.quant;+ gf.dst = (uint64_t) (uintptr_t) f.dst;+ gf.dst_stride = f.dst_stride;+ gf.pdst = (uint64_t) (uintptr_t) f.pdst;+ gf.pdst_stride = f.pdst_stride;++ mtl::Pipeline pl;+ try {+ pl = g_cx->pipeline(g_lib, pack ? "ds4_finish_pack" : "ds4_finish");+ } catch (...) {+ return false;+ }+ // one dispatch, one threadgroup a vector (the kernel's blockIdx.x is threadgroup_position_in_grid)+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bx, off), mtl::Arg::V((long) stride), mtl::Arg::V((unsigned) len), mtl::Arg::V(gf)};+ d.grid[0] = (uint32_t) n_vec;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256;+ d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ // The fp8-rounding finishes (the KV rows, the attention output) are read stale by the next dispatch when it shares their+ // command buffer: end the buffer after them (committed, not waited for: the queue orders the buffers).+ // (it once ended the command buffer after every fp8-rounding finish, 2 a layer, against a stale read; each extra buffer costs the GPU ~0.3 ms idle, a third of a decode token.+ // The decode-path score is bit-identical without it, the dispatch barrier after every dispatch orders the readers. DS4_FINISH_SPLIT=1 brings it back.)+ static const bool split_after = std::getenv("DS4_FINISH_SPLIT") != nullptr;+ if (f.quant && split_after) g_cx->split_runs();+ if (const char* fm2 = std::getenv("DS4_FLUSH_FINISH_CTX")) { if (f.quant) { if (std::atoi(fm2) == 2) g_cx->split_runs(); else g_cx->flush_runs(); } }+ if (const char* fm = std::getenv("DS4_FLUSH_FINISH")) { // debugging: which finish calls need a drain after them+ const int m = std::atoi(fm);+ if ((m == 1 && f.dst) || (m == 2 && f.rd > 0) || (m == 3 && f.w) || (m == 4 && f.quant) || (m == 5 && !f.dst) ||+ (m == 6 && f.dst == nullptr && f.rd == 0) || (m == 7 && f.rd > 0 && f.inverse) || (m == 8 && f.rd > 0 && !f.inverse && !f.dst) || (m == 9 && f.rd > 0 && !f.dst) || (m == 10 && (f.dst || f.inverse)) || (m == 11 && f.rd > 0 && !f.inverse))+ ds4_metal_flush();+ }+ return true;+ }+++ // ds4_kernels.cu:1677 - hc_post_inplace: h[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*h_old[t][j][e]+ bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T) {+ if (!ensure()) return false;+ void* hy = strata::gpu::host_view((void*) y);+ void* hh = strata::gpu::host_view(h);+ if (!hy || !hh) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );+ if (!by.handle || !bh.handle) return false;+ // post is [T][8], comb is [T][64]: device operands like the others (reached through their host views)+ void* hpost = strata::gpu::host_view((void*) post);+ void* hcomb = strata::gpu::host_view((void*) comb);+ if (!hpost || !hcomb) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view for post / comb\n"); return false; }+ mtl::Buffer bp = wrap(hpost, (size_t) T * 8 * 4);+ mtl::Buffer bc = wrap(hcomb, (size_t) T * 64 * 4);+ if (!bp.handle || !bc.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_post"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bp, off(hpost)), mtl::Arg::B(bc, off(hcomb)),+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};+ d.grid[0] = (uint32_t) ((dim + 255) / 256);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ const bool dbg = std::getenv("DS4_DBG_SG") != nullptr;+ if (dbg) {+ g_cx->flush_runs();+ const float* fy = (const float*) hy; const float* fh = (const float*) hh; const float* fp = (const float*) hpost; const float* fc = (const float*) hcomb;+ std::fprintf(stderr, "hc_post in: y %g %g h %g %g | %g %g post %g %g %g %g comb %g %g %g %g\n", fy[0], fy[1], fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1], fp[0], fp[1], fp[2], fp[3], fc[0], fc[1], fc[2], fc[3]);+ }+ g_cx->run({d});+ if (dbg) {+ g_cx->flush_runs();+ const float* fh = (const float*) hh;+ std::fprintf(stderr, "hc_post out: h %g %g | %g %g\n", fh[0], fh[1], fh[(size_t) dim], fh[(size_t) dim + 1]);+ }+ g_cx->free(bp);+ g_cx->free(bc);+ return true;+ }+++ // ds4_kernels.cu:1692 - hc_mix_scale: mixes[t][i] *= rsqrt(sum(h[t])^2 / hcdim + eps)+ bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes) {+ if (!ensure()) return false;+ void* hh = strata::gpu::host_view((void*) h);+ void* hm = strata::gpu::host_view((void*) mixes);+ if (!hh || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix_scale has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );+ if (!bh.handle || !bm.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_scale"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::V((unsigned) hcdim),+ mtl::Arg::V((unsigned) n_mix), mtl::Arg::V(eps)};+ d.grid[0] = (uint32_t) T;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ if (std::getenv("DS4_DBG_SG")) {+ g_cx->flush_runs();+ const float* fc = (const float*) hm;+ std::fprintf(stderr, "hc_mix_scale T %d hcdim %d n_mix %d eps %g -> %g %g %g\n", T, hcdim, n_mix, eps, fc[0], fc[1], fc[2]);+ }+ return true;+ }+++ // ds4_kernels.cu:1901 - hc_mix: mixes[t][m] = (h[t] . fn[m]) * rsqrt(mean(h[t]^2) + eps)+ bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes) {+ if (!ensure()) return false;+ void* hh = strata::gpu::host_view((void*) h);+ void* hf = strata::gpu::host_view((void*) fn);+ void* hm = strata::gpu::host_view((void*) mixes);+ if (!hh || !hf || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 );+ mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4 );+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );+ if (!bh.handle || !bf.handle || !bm.handle) return false;+ mtl::Pipeline pl;+ const bool all = T >= 16 && n_mix <= 32 && !std::getenv("DS4_NO_HC_ALL"); // a prompt chunk: a threadgroup a token, 4 mixes a pass+ try { pl = g_cx->pipeline(g_lib, all ? "ds4_hc_mix_all" : "ds4_hc_mix"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bf, off(hf)), mtl::Arg::B(bm, off(hm)),+ mtl::Arg::V((unsigned) hcdim), mtl::Arg::V(eps)};+ if (all) d.args.push_back(mtl::Arg::V((unsigned) n_mix));+ d.grid[0] = all ? (uint32_t) T : (uint32_t) n_mix;+ d.grid[1] = all ? 1u : (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }+++ // ds4_kernels.cu:1908 - hc_split: the sinkhorn normalisation (scalar path; see the MSL comment)+ bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,+ float* pre, float* post, float* comb) {+ if (!ensure()) return false;+ void* hm = strata::gpu::host_view((void*) mixes);+ void* hp = strata::gpu::host_view((void*) pre);+ void* ho = strata::gpu::host_view((void*) post);+ void* hc_ = strata::gpu::host_view((void*) comb);+ if (!hm || !hp || !ho || !hc_) { std::fprintf(stderr, "dsv41 metal: hc_split has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 );+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );+ mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4 );+ mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4 );+ if (!bm.handle || !bp.handle || !bo.handle || !bc.handle) return false;+ // scale and base come from the engine like every other operand: reach them through host_view, not as raw host pointers+ void* hs = strata::gpu::host_view((void*) scale);+ void* hb = strata::gpu::host_view((void*) base);+ if (!hs || !hb) { std::fprintf(stderr, "dsv41 metal: hc_split scale/base have no host view\n"); return false; }+ const size_t nbase = (size_t) (2 * hc + hc * hc);+ mtl::Buffer bs = wrap(hs, 3 * sizeof(float));+ mtl::Buffer bb = wrap(hb, nbase * sizeof(float));+ if (!bs.handle || !bb.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, hc == 4 && !std::getenv("DS4_NO_HCS4") ? "ds4_hc_split4" : "ds4_hc_split"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bp, off(hp)),+ mtl::Arg::B(bo, off(ho)), mtl::Arg::B(bc, off(hc_)), mtl::Arg::V((unsigned) hc),+ mtl::Arg::V((unsigned) iters), mtl::Arg::V((float) hc_eps)};+ d.grid[0] = (uint32_t) T;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 32; d.tg[1] = d.tg[2] = 1;+ if (std::getenv("DS4_DBG_SG")) {+ g_cx->flush_runs();+ const float* fm = (const float*) hm; const float* fs = (const float*) hs; const float* fb = (const float*) hb;+ std::fprintf(stderr, "hc_split in: mix %g %g %g scale %g %g %g base %g %g\n", fm[0], fm[1], fm[2], fs[0], fs[1], fs[2], fb[0], fb[1]);+ }+ g_cx->run({d});+ if (std::getenv("DS4_DBG_SG")) {+ g_cx->flush_runs();+ const float* fp = (const float*) hp;+ std::fprintf(stderr, "hc_split out: pre %g %g %g %g\n", fp[0], fp[1], fp[2], fp[3]);+ }+ return true;+ }++++// the hyper-connection mix and split in one dispatch (ds4_hc_mix_split): the engine used to issue hc_mix and hc_split++// back to back for the same token, and the kernel does both in one threadgroup, bit for bit as before.++bool hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes,++ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,++ float* comb) {++ if (!ensure()) return false;++ void* hh = strata::gpu::host_view((void*) h);++ void* hf = strata::gpu::host_view((void*) fn);++ void* hm = strata::gpu::host_view((void*) mixes);++ void* hs = strata::gpu::host_view((void*) scale);++ void* hb = strata::gpu::host_view((void*) base);++ void* hp = strata::gpu::host_view((void*) pre);++ void* ho = strata::gpu::host_view((void*) post);++ void* hc_ = strata::gpu::host_view((void*) comb);++ if (!hh || !hf || !hm || !hs || !hb || !hp || !ho || !hc_) {++ std::fprintf(stderr, "dsv41 metal: hc_mix_split has no host view\n");++ return false;++ }++ const size_t page = 16384;++ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)++ const size_t base_ = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);++ const size_t end = (size_t) ((uintptr_t) host) + bytes;++ return g_cx->wrap((void*) base_, ((end - base_ + page - 1) / page) * page);++ };++ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(size_t) (page - 1))); };++ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4);++ mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4);++ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4);++ mtl::Buffer bs = wrap(hs, 3 * sizeof(float));++ mtl::Buffer bb = wrap(hb, (size_t) (2 * hc + hc * hc) * sizeof(float));++ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4);++ mtl::Buffer bo = wrap(ho, (size_t) T * 8 * 4);++ mtl::Buffer bc = wrap(hc_, (size_t) T * 64 * 4);++ if (!bh.handle || !bf.handle || !bm.handle || !bs.handle || !bb.handle || !bp.handle || !bo.handle || !bc.handle)++ return false;++ mtl::Pipeline pl;++ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_split"); } catch (...) { return false; }++ mtl::Dispatch d;++ d.p = &pl;++ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bf, off(hf)), mtl::Arg::B(bm, off(hm)),++ mtl::Arg::V((unsigned) hcdim), mtl::Arg::V((float) eps), mtl::Arg::V((unsigned) n_mix),++ mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bp, off(hp)),++ mtl::Arg::B(bo, off(ho)), mtl::Arg::B(bc, off(hc_)), mtl::Arg::V((unsigned) hc),++ mtl::Arg::V((unsigned) iters), mtl::Arg::V((float) hc_eps)};++ d.grid[0] = (uint32_t) T;++ d.grid[1] = d.grid[2] = 1;++ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;++ g_cx->run({d});++ return true;++}+++++ // ds4_kernels.cu:1938 - moe_sort: count each virtual slot's picks, prefix-sum the counts, fill in token order.+ // Three dispatches on one command buffer, so their order is the order of the calls (serial dispatch).+ bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos) {+ if (!ensure()) return false;+ void* hi = strata::gpu::host_view((void*) idx);+ void* hv = strata::gpu::host_view((void*) vs);+ void* hb = strata::gpu::host_view((void*) bounds);+ void* hr = strata::gpu::host_view((void*) row_tok);+ void* hp = strata::gpu::host_view((void*) pos);+ if (!hi || !hv || !hb || !hr || !hp) { std::fprintf(stderr, "dsv41 metal: moe_sort has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ const size_t t_bytes = (size_t) T * (size_t) k * 4;+ mtl::Buffer bi = wrap(hi, t_bytes);+ mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 );+ mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 );+ mtl::Buffer br = wrap(hr, t_bytes);+ mtl::Buffer bp = wrap(hp, t_bytes);+ if (!bi.handle || !bv.handle || !bb.handle || !br.handle || !bp.handle) return false;+ mtl::Pipeline pc, pf, ps;+ try {+ pc = g_cx->pipeline(g_lib, "ds4_moe_sort_count");+ pf = g_cx->pipeline(g_lib, "ds4_moe_sort_fill");+ ps = g_cx->pipeline(g_lib, "ds4_bounds_scan");+ } catch (...) { return false; }+ // 1. counts+ mtl::Dispatch d1;+ d1.p = &pc;+ d1.args = {mtl::Arg::B(bi, off(hi)), mtl::Arg::V((unsigned) T), mtl::Arg::V((unsigned) k),+ mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bb, off(hb))};+ d1.grid[0] = (uint32_t) n_vs;+ d1.grid[1] = d1.grid[2] = 1;+ d1.tg[0] = 256; d1.tg[1] = d1.tg[2] = 1;+ g_cx->run({d1});+ // 2. the exclusive prefix over the slots+ mtl::Dispatch d2;+ d2.p = &ps;+ d2.args = {mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n_vs)};+ d2.grid[0] = 1;+ d2.grid[1] = d2.grid[2] = 1;+ d2.tg[0] = 32; d2.tg[1] = d2.tg[2] = 1;+ g_cx->run({d2});+ // 3. the fill+ mtl::Dispatch d3;+ d3.p = &pf;+ d3.args = {mtl::Arg::B(bi, off(hi)), mtl::Arg::V((unsigned) T), mtl::Arg::V((unsigned) k),+ mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp))};+ d3.grid[0] = (uint32_t) n_vs;+ d3.grid[1] = d3.grid[2] = 1;+ d3.tg[0] = 256; d3.tg[1] = d3.tg[2] = 1;+ g_cx->run({d3});+ return true;+ }+++ // ds4_kernels.cu:1690 - the Engram row decoder (fmt 2 = IQ4_NL, this model's tables)+ bool engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, int fmt) {+ if (T <= 0) return true;+ if (!ensure()) return false;+ void* hs = strata::gpu::host_view((void*) src);+ void* ho = strata::gpu::host_view((void*) out);+ if (!hs || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant_tok has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ const size_t rows = (size_t) T * (size_t) cols;+ mtl::Buffer bs = wrap(hs, (size_t) T * per );+ mtl::Buffer bo = wrap(ho, rows * (size_t) hd * 4 );+ if (!bs.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((unsigned long) per), mtl::Arg::V((unsigned) cols),+ mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((int) fmt)};+ d.grid[0] = (uint32_t) rows;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }+++ // ds4_kernels.cu:341 - hc_pre_norm: a = sum_s mix[s]*h[s][e]; x = a * rsqrt(mean(a^2) + eps) (x w when given)+ bool hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x) {+ if (!ensure()) return false;+ void* hh = strata::gpu::host_view((void*) h);+ void* hm = strata::gpu::host_view((void*) mix);+ void* hx = strata::gpu::host_view((void*) x);+ if (!hh || !hm || !hx) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 );+ mtl::Buffer bm = wrap(hm, (size_t) T * 8 * 4 );+ mtl::Buffer bx = wrap(hx, (size_t) T * dim * 4 );+ if (!bh.handle || !bm.handle || !bx.handle) return false;+ mtl::Buffer bw;+ size_t w_off = 0;+ if (w) {+ void* hw = strata::gpu::host_view((void*) w);+ if (!hw) { std::fprintf(stderr, "dsv41 metal: hc_pre_norm weight has no host view\n"); return false; }+ bw = wrap(hw, (size_t) dim * 4 );+ w_off = off(hw);+ if (!bw.handle) return false;+ } else {+ bw = bx; // never read when has_w == 0+ }+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_pre_norm"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bw, w_off),+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc), mtl::Arg::V(eps), mtl::Arg::B(bx, off(hx)),+ mtl::Arg::V((unsigned) (w ? 1 : 0))};+ d.grid[0] = (uint32_t) T;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }+++ // ds4_gguf.cu:186 - the activation quantiser the expert matmuls consume (Q8_0: 32 values, one scale)+ bool q8_quant_rows(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {+ if (rows <= 0 || k <= 0) return true;+ if (!ensure()) return false;+ void* hx = strata::gpu::host_view((void*) x);+ void* hq = strata::gpu::host_view((void*) q);+ void* hd = strata::gpu::host_view((void*) d);+ if (!hx || !hq || !hd) { std::fprintf(stderr, "dsv41 metal: q8_quant has no host view\n"); return false; }+ const size_t page = 16384;+ // The pages the buffer itself spans: [base, base + off + bytes). A fixed slack (the +65536 the call sites used to+ // carry) walks off the end of the mapping when the buffer is small and sits at an arena chunk's edge, and Metal then+ // refuses the no-copy wrap ("mtl: wrap 81920 failed" - the hd scales here, a few pages).+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bx = wrap(hx, (size_t) rows * ldx * 4);+ mtl::Buffer bq = wrap(hq, (size_t) rows * k);+ mtl::Buffer bd = wrap(hd, (size_t) rows * (k / 32) * 4);+ if (!bx.handle || !bq.handle || !bd.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) k),+ mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bd, off(hd))};+ const uint groups = (uint32_t) ((k / 32 + 7) / 8);+ dsp.grid[0] = groups;+ dsp.grid[1] = (uint32_t) rows;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }+++ // the GPU address of host weight memory (see mtl::Context::zc_map); 0: not available+ uint64_t zc_map(const void* p, size_t bytes) {+ if (!ensure()) return 0;+ return g_cx->zc_map(p, bytes);+ }++ // Several ds4_q8_quant calls in one dispatch (specs: {x, ldx, rows, k, q, d} with the engine's device pointers, which are the+ // kernels' own addresses, as iq_mv's jobs carry them)+ namespace {+ struct HostQ8Spec {+ const float* x;+ uint64_t ldx;+ uint32_t rows, k;+ int8_t* q;+ float* d;+ };+ } // namespace+ bool q8_quant_multi(const void* specs, int n) {+ if (n <= 0) return true;+ if (!ensure()) return false;+ const HostQ8Spec* sp = (const HostQ8Spec*) specs;+ uint32_t max_rows = 1, max_k = 32;+ for (int i = 0; i < n; ++i) {+ max_rows = std::max(max_rows, sp[i].rows);+ max_k = std::max(max_k, sp[i].k);+ }+ mtl::Buffer bj = g_cx->alloc((size_t) n * sizeof(HostQ8Spec));+ std::memcpy(bj.ptr, sp, (size_t) n * sizeof(HostQ8Spec));+ static mtl::Pipeline pl;+ static bool have = false;+ if (!have) {+ try { pl = g_cx->pipeline(g_lib, "ds4_q8_quant_multi"); } catch (...) { g_cx->free(bj); return false; }+ have = true;+ }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (max_k / 32 + 7) / 8;+ d.grid[1] = max_rows;+ d.grid[2] = (uint32_t) n;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+ }+++ // ds4_gguf.cu:371 - the MXFP4 expert matvec. The engine builds its BfJobs array on the HOST (with device pointers inside), so+ // the jobs go to the kernel as an opaque constant copy: same bytes, same layout, indexed by threadgroup_position_in_grid.y.+ namespace {+ // mirrors ds4_gguf.cu's BfJob (and the kernels' Ds4MxJob)+ struct HostBfJob {+ const void* w;+ uint64_t ld;+ const float* x;+ uint64_t xcs;+ float* y;+ uint64_t ycs;+ const float* bias;+ float alpha;+ int k, n, ncol;+ const int8_t* xq;+ const float* xd;+ };+ } // namespace++ bool mx_mv(const void* jobs_in, int nj_in, int max_n, size_t job_bytes, int fmt) {+ if (nj_in <= 0 || max_n <= 0) return true;+ if (!ensure()) return false;+ // The matvec kernels keep 8 accumulators (the T=8 window): a job of more activations (a 16-token window) is cut into+ // jobs of at most 8, each reading its own activations and writing its own rows of y. (Left whole, the columns past the+ // eighth were never computed: rows of zeros in every projection of a prompt window.)+ std::vector<HostBfJob> cut;+ if (job_bytes == sizeof(HostBfJob)) {+ const HostBfJob* src = (const HostBfJob*) jobs_in;+ for (int i = 0; i < nj_in; ++i) {+ const HostBfJob& q = src[i];+ const int ncol = q.ncol > 0 ? q.ncol : 1;+ for (int c0 = 0; c0 < ncol; c0 += 8) {+ HostBfJob b = q;+ b.ncol = std::min(8, ncol - c0);+ if (c0 != 0) {+ if (q.x) b.x = q.x + (size_t) c0 * (size_t) q.xcs;+ b.y = q.y + (size_t) c0 * (size_t) q.ycs;+ if (q.xq) b.xq = q.xq + (size_t) c0 * (size_t) q.k;+ if (q.xd) b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);+ }+ cut.push_back(b);+ }+ }+ }+ const void* jobs = cut.empty() ? jobs_in : (const void*) cut.data();+ const int nj = cut.empty() ? nj_in : (int) cut.size();+ mtl::Buffer bj = g_cx->alloc((size_t) nj * job_bytes);+ if (!bj.ptr) { std::fprintf(stderr, "dsv41 metal: mx_mv job buffer failed\n"); return false; }+ std::memcpy(bj.ptr, jobs, (size_t) nj * job_bytes);+ mtl::Pipeline pl;+ // fmt 1 = MXFP4, 2 = Q8_0, 0 = BF16 (a dense matrix: the engram's wkv, ds4_gguf.cu:1064)+ const char* fn = fmt == 2 ? "ds4_q8_mv" : (fmt == 1 ? "ds4_mx_mv" : "ds4_bf16_mv");+ try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);+ d.grid[1] = (uint32_t) nj;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+ }+++ // ds4_iq.cu:809 - the I-quant expert matvec. The jobs are the engine's own MvJob array (80 bytes each, type included), so they+ // travel as an opaque constant copy and the kernel picks the decoder per job; one launch covers every type in it.+ static inline void* cpu_addr(void* p);+ namespace {+ // mirrors ds4_iq.cuh's MvJob (and ds4_iq.metal's Ds4MvJob): 80 bytes on arm64+ struct Ds4IqMvJob {+ const uint8_t* w;+ uint64_t ld;+ int type, k, n, ncol;+ const int8_t* xq;+ const float* xd;+ float* y;+ uint64_t ycs;+ const float* bias;+ float alpha;+ };+ static_assert(sizeof(Ds4IqMvJob) == 80, "Ds4IqMvJob layout must match ds4_iq.cuh's MvJob");+ } // namespace++ bool iq_mv(const void* jobs, int nj) {+ if (nj <= 0) return true;+ if (!ensure()) return false;+ const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;+ // Each job goes to the kernel of its type and column count: NC = 1, 2, 4 or 8 activations decode the row's weights once+ // (ds4_*_mvn<NC>); a job of more columns is cut into pieces of 8. Other types keep the 16-column kernels.+ std::map<std::string, std::vector<Ds4IqMvJob>> groups;+ int max_n = 0;+ if (std::getenv("DS4_DBG_JOBS")) {+ std::fprintf(stderr, "iq_mv call:");+ for (int i = 0; i < nj; ++i) std::fprintf(stderr, " [t%d n%d k%d c%d]", src[i].type, src[i].n, src[i].k, src[i].ncol);+ std::fprintf(stderr, "\n");+ }+ static const bool old_only = std::getenv("DS4_NO_MVN") != nullptr;+ for (int i = 0; i < nj; ++i) {+ const Ds4IqMvJob& q = src[i];+ { static const char* sk = std::getenv("DS4_SKIP_TYPES"); if (sk != nullptr && std::strstr(sk, ("," + std::to_string(q.type) + ",").c_str()) != nullptr) continue; } // (timing only)+ if (q.n > max_n) max_n = q.n;+ const int ncol = q.ncol > 0 ? q.ncol : 1;+ const char* tn = nullptr;+ switch (q.type) {+ case 13: tn = "ds4_q5k_mvn"; break;+ case 16: tn = "ds4_iq2xxs_mvn"; break;+ case 17: tn = "ds4_iq2xs_mvn"; break;+ case 18: tn = "ds4_iq3xxs_mvn"; break;+ case 22: tn = "ds4_iq2s_mvn"; break;++ case 29: tn = "ds4_iq1m_mvn"; break; // (the 2.00 bpw pack's IQ1_M: ds4_iq.metal's ds4_w32_iq1m)+ default: break;+ }+ const bool grid = tn != nullptr && !old_only;+ const int step = grid ? 8 : (ncol > 16 ? 16 : ncol);+ for (int c0 = 0; c0 < ncol; c0 += step) {+ Ds4IqMvJob b = q;+ b.ncol = (c0 + step < ncol) ? step : ncol - c0;+ if (c0 != 0) {+ b.xq = q.xq + (size_t) c0 * (size_t) q.k;+ b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);+ b.y = q.y + (size_t) c0 * (size_t) q.ycs;+ }+ std::string fn;+ if (grid) {+ const int nc = b.ncol <= 1 ? 1 : (b.ncol <= 2 ? 2 : (b.ncol <= 4 ? 4 : 8));+ fn = std::string(tn) + std::to_string(nc);+ } else {+ fn = (b.type == 13 || b.type == 14) ? "ds4_iq_mv_k" : "ds4_iq_mv";+ }+ groups[fn].push_back(b);+ }+ }+ if (max_n <= 0) return true;+ const size_t kBatch = 64; // the CUDA host's job-array limit; one command buffer a group+ for (auto& kv : groups) {+ std::vector<Ds4IqMvJob>& v = kv.second;+ mtl::Buffer bj = g_cx->alloc(v.size() * sizeof(Ds4IqMvJob));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: iq_mv job buffer failed\n");+ return false;+ }+ std::memcpy(bj.ptr, v.data(), v.size() * sizeof(Ds4IqMvJob));+ mtl::Pipeline pl;+ try {+ pl = g_cx->pipeline(g_lib_iq, kv.first);+ } catch (...) {+ g_cx->free(bj);+ return false;+ }+ std::vector<mtl::Dispatch> ds;+ for (size_t base = 0; base < v.size(); base += kBatch) {+ const size_t cnt = std::min(kBatch, v.size() - base);+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj, base * sizeof(Ds4IqMvJob))};+ d.grid[0] = (uint32_t) ((max_n + 7) / 8);+ d.grid[1] = (uint32_t) cnt;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ ds.push_back(d);+ }+ g_cx->run(ds);+ g_cx->free(bj);+ }+ return true;+ }+++ // ds4_iq.cu:536 gate_up_kernel<T, NE> / :588 down_kernel<T, NE> - the prompt / window path's expert GEMM (the launches at+ // :958 launch_gu / :960 launch_dn). A threadgroup of 256 is 8 warps: a warp RW rows (gate / up 1, down 2), a lane the+ // row's sub-blocks, the group's entries four at a pass. The group's blob and its entries are read the way the CUDA kernel+ // reads them, so the plan / window pointer arrays travel as they are: every pointer here is a CPU address (unified memory)+ // and a kernel may take raw gpuAddresses (as ds4_finish's f.w does).+ namespace {+ // (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4IqJob (Metal has no 'long long': 64-bit is 'long')+ struct IqJob {+ const void* plan_ptr;+ const void* grp_ptr;+ const void* grp_start;+ const void* grp_ent;+ const void* n_groups;+ const void* xq;+ const void* xd;+ const void* hq;+ const void* hd;+ void* h;+ void* part;+ int64_t tr0, tr1, tr2;+ int n_embd, n_ff, type;+ float limit;+ };++ static bool iq_gemm(const IqJob& j, const char* fn, int grid_x, int groups) {+ if (!ensure()) return false;+ mtl::Buffer bj = g_cx->alloc(sizeof(j));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);+ return false;+ }+ std::memcpy(bj.ptr, &j, sizeof(j));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) grid_x;+ d.grid[1] = (uint32_t) groups;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+ }+ } // namespace++ // gate / up of the group's entries: h = the clamped swiglu (ds4_iq.cuh's Args: tr[0] gate, tr[1] up, tr[2] down)+ bool iq_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,+ const int8_t* xq, const float* xd, float* h, int type, int groups) {+ if (groups <= 0) return true;+ if (n_embd <= 0 || n_ff <= 0 || n_embd % 256 != 0 || n_ff % 8 != 0) return false;+ if (xq == nullptr || xd == nullptr || h == nullptr) return false;+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;+ IqJob j{};+ j.plan_ptr = plan_ptr;+ j.grp_ptr = grp_ptr;+ j.grp_start = grp_start;+ j.grp_ent = grp_ent;+ j.n_groups = n_groups;+ j.xq = xq;+ j.xd = xd;+ j.h = h;+ j.tr0 = (int64_t) tr[0];+ j.tr1 = (int64_t) tr[1];+ j.tr2 = (int64_t) tr[2];+ j.n_embd = n_embd;+ j.n_ff = n_ff;+ j.type = type;+ j.limit = limit;+ return iq_gemm(j, "ds4_iq_gu", n_ff / 8, groups);+ }++ // down of the group's entries against their int8 h: part (a row of n_embd an entry)+ bool iq_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,+ const float* hd, float* part, int type, int groups) {+ if (groups <= 0) return true;+ if (n_embd <= 0 || n_ff <= 0 || n_ff % 256 != 0 || n_embd % 16 != 0) return false;+ if (hq == nullptr || hd == nullptr || part == nullptr) return false;+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;+ IqJob j{};+ j.plan_ptr = plan_ptr;+ j.grp_ptr = grp_ptr;+ j.grp_start = grp_start;+ j.grp_ent = grp_ent;+ j.n_groups = n_groups;+ j.hq = hq;+ j.hd = hd;+ j.part = part;+ j.tr0 = (int64_t) tr[0];+ j.tr1 = (int64_t) tr[1];+ j.tr2 = (int64_t) tr[2];+ j.n_embd = n_embd;+ j.n_ff = n_ff;+ j.type = type;+ j.limit = 0.0f;+ return iq_gemm(j, "ds4_iq_dn", n_embd / 16, groups);+ }+++ // ds4_gguf.cu:454 mx_gate_up_kernel / :519 mx_down_kernel - the MXFP4 expert GEMM (ds4_gguf.cu's moe_gate_up / moe_down /+ // win_experts). The same frame as iq_gu / iq_dn above: a threadgroup of 256 is 8 warps, a warp a row, a lane strides the+ // 32-value blocks, the group's entries sixteen at a pass. The job struct is the kernel's own (metal/src/engine/msl/+ // ds4_iq.metal's Ds4MxJob); the plan / window pointer arrays travel as they are: every pointer here is a CPU address+ // (unified memory) and a kernel may take raw gpuAddresses.+ namespace {+ // (not an entry point: the generator takes only the ^bool lines) mirrors metal/src/engine/msl/ds4_iq.metal's Ds4MxJob (Metal+ // has no 'long long': 64-bit is 'long')+ struct MxJob {+ const void* plan_ptr;+ const void* grp_ptr;+ const void* grp_start;+ const void* grp_ent;+ const void* n_groups;+ const void* xq;+ const void* xd;+ const void* hq;+ const void* hd;+ void* h;+ void* part;+ int64_t tr0, tr1, tr2; // ('long' there, so 8 a field and the float ends the struct on the same byte)+ int n_embd, n_ff;+ float limit;+ };+ static_assert(sizeof(MxJob) == 128, "Ds4MxJob's layout");++ static bool mx_gemm(const MxJob& j, const char* fn, int grid_x, int groups) {+ if (!ensure()) return false;+ mtl::Buffer bj = g_cx->alloc(sizeof(j));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: %s job buffer failed\n", fn);+ return false;+ }+ std::memcpy(bj.ptr, &j, sizeof(j));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib_iq, fn); } catch (...) { g_cx->free(bj); return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) grid_x;+ d.grid[1] = (uint32_t) groups;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+ }+ } // namespace++ // gate / up of the group's experts: h = the clamped swiglu (ds4_gguf.cu's MxArgs: tr[0] gate, tr[1] up, tr[2] down)+ bool mx_gu(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, float limit,+ const int8_t* xq, const float* xd, float* h, int groups) {+ if (groups <= 0) return true;+ if (n_embd <= 0 || n_ff <= 0 || n_embd % 32 != 0 || n_ff % 8 != 0) return false;+ if (xq == nullptr || xd == nullptr || h == nullptr) return false;+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;+ MxJob j{};+ j.plan_ptr = plan_ptr;+ j.grp_ptr = grp_ptr;+ j.grp_start = grp_start;+ j.grp_ent = grp_ent;+ j.n_groups = n_groups;+ j.xq = xq;+ j.xd = xd;+ j.h = h;+ j.tr0 = (int64_t) tr[0];+ j.tr1 = (int64_t) tr[1];+ j.tr2 = (int64_t) tr[2];+ j.n_embd = n_embd;+ j.n_ff = n_ff;+ j.limit = limit;+ return mx_gemm(j, "ds4_mx_gu", (n_ff + 7) / 8, groups);+ }++ // down of the group's experts against their int8 h: part (a row of n_embd an entry)+ bool mx_dn(const unsigned long long* plan_ptr, const unsigned long long* grp_ptr, const int* grp_start,+ const int* grp_ent, const int* n_groups, const size_t* tr, int n_embd, int n_ff, const int8_t* hq,+ const float* hd, float* part, int groups) {+ if (groups <= 0) return true;+ if (n_embd <= 0 || n_ff <= 0 || n_ff % 32 != 0 || n_embd % 8 != 0) return false;+ if (hq == nullptr || hd == nullptr || part == nullptr) return false;+ if (plan_ptr == nullptr && grp_ptr == nullptr) return false;+ MxJob j{};+ j.plan_ptr = plan_ptr;+ j.grp_ptr = grp_ptr;+ j.grp_start = grp_start;+ j.grp_ent = grp_ent;+ j.n_groups = n_groups;+ j.hq = hq;+ j.hd = hd;+ j.part = part;+ j.tr0 = (int64_t) tr[0];+ j.tr1 = (int64_t) tr[1];+ j.tr2 = (int64_t) tr[2];+ j.n_embd = n_embd;+ j.n_ff = n_ff;+ return mx_gemm(j, "ds4_mx_dn", (n_embd + 7) / 8, groups);+ }++ // ds4_gguf.cu:577 mx_combine_kernel - a token's expert outputs summed in plan order (the experts off the device add 0),+ // the CPU lane's part when its flag is set, then the shared expert's: ds4_gguf.cu:169's combine(), 128 threads a block+ namespace {+ // mirrors metal/src/engine/msl/ds4_init.metal's Ds4Comb (a 'long' there is int64_t here)+ struct MxCombJob {+ const void* plan_ptr;+ const void* plan_w;+ const void* part;+ const void* cpu_part;+ const int* cpu_flag;+ const void* sh_out;+ void* out;+ int64_t cpu_stride, out_stride;+ int k, n_embd;+ };+ } // namespace++ bool mx_combine(const void* plan_ptr, const void* plan_w, const void* part, const void* cpu_part, size_t cpu_stride,+ const int* cpu_flag, const void* sh_out, void* out, size_t out_stride, int T, int k, int n_embd) {+ if (T <= 0 || n_embd <= 0) return true;+ if (k <= 0 || plan_ptr == nullptr || plan_w == nullptr || part == nullptr || out == nullptr) return false;+ if (!ensure()) return false;+ MxCombJob j{};+ j.plan_ptr = plan_ptr;+ j.plan_w = plan_w;+ j.part = part;+ j.cpu_part = cpu_part;+ j.cpu_flag = cpu_flag;+ j.sh_out = sh_out;+ j.out = out;+ j.cpu_stride = (int64_t) cpu_stride;+ j.out_stride = (int64_t) out_stride;+ j.k = k;+ j.n_embd = n_embd;+ mtl::Buffer bj = g_cx->alloc(sizeof(j));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: mx_combine job buffer failed\n");+ return false;+ }+ std::memcpy(bj.ptr, &j, sizeof(j));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_mx_combine"); } catch (...) { g_cx->free(bj); return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) ((n_embd + 127) / 128);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+ }++ // ds4_kernels.cu:1092 - the KV ring row to BF16 (E4M3 code x E8M0 group scale)+ bool ring_to_ext(const void* ring, void* ext, long long p0, int n, int d, size_t row_bytes) {+ if (n <= 0 || d <= 0) return true;+ if (!ensure()) return false;+ void* hr = strata::gpu::host_view((void*) ring);+ void* he = strata::gpu::host_view(ext);+ if (!hr || !he) { std::fprintf(stderr, "dsv41 metal: ring_to_ext has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );+ mtl::Buffer be = wrap(he, (size_t) n * d * 2 );+ if (!br.handle || !be.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_ring_to_ext"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::B(be, off(he)), mtl::Arg::V((long) p0),+ mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes)};+ dsp.grid[0] = (uint32_t) n;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }+++ // ds4_kernels.cu:1104 - the window back into the ring (E4M3 codes + E8M0 group scales, optional backup)+ bool ext_to_ring(const void* ext, void* ring, long long p0, int n, int T, int d, size_t row_bytes, void* backup) {+ if (T <= 0 || n <= 0 || d <= 0) return true;+ if (!ensure()) return false;+ void* he = strata::gpu::host_view((void*) ext);+ void* hr = strata::gpu::host_view(ring);+ if (!he || !hr) { std::fprintf(stderr, "dsv41 metal: ext_to_ring has no host view\n"); return false; }+ void* hb = backup ? strata::gpu::host_view(backup) : nullptr;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer be = wrap(he, (size_t) (n + T) * d * 2 );+ mtl::Buffer br = wrap(hr, (size_t) n * row_bytes );+ if (!be.handle || !br.handle) return false;+ mtl::Buffer bb = br; // never written when backup is null+ size_t b_off = 0;+ if (hb) { bb = wrap(hb, (size_t) T * row_bytes ); b_off = off(hb); if (!bb.handle) return false; }+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_ext_to_ring"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bb, b_off),+ mtl::Arg::V((long) p0), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) T),+ mtl::Arg::V((unsigned) d), mtl::Arg::V((unsigned long) row_bytes), mtl::Arg::V((unsigned) (hb ? 1 : 0))};+ dsp.grid[0] = (uint32_t) T;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }+++ // ds4_kernels.cu:937 - the selected compressed rows into the scratch (MXFP4 -> BF16, 16 threads a row)++ namespace {+ // mirrors metal/src/engine/msl/ds4_init.metal's Ds4AttnRows (ds4_kernels.cuh:149 AttnRows, kWin = 16)+ struct Ds4AttnRowsJob {+ int win0[16], nwin[16], nsel[16];+ int gen, t0, win, R, kcap;+ int64_t p0;+ };+ static_assert(sizeof(Ds4AttnRowsJob) == 224, "Ds4AttnRowsJob layout must match the MSL struct");+ } // namespace++ // ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false> - the prompt path's sparse attention.+ // The packed kernel stages rows once for G heads of a query; the BF16 kernel handles rows already in BF16 (a DSpark block).+ bool sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int* sel, int sel_stride, const void* rows,+ const float* sink, int heads, int d, int T, float scale, float* out, int ckv_fmt) {+ if (T <= 0 || heads <= 0) return true;+ if (d != 512 || rows == nullptr || (ckv_fmt != 0 && ckv_fmt != 2)) { std::fprintf(stderr, "dsv41 metal: sparse_attn unsupported\n"); return false; }+ Ds4AttnRowsJob ar{};++ ar.t0 = (int) T; // the two-query kernel reads the row count here on the gen == 0 path+ std::memcpy(&ar, rows, sizeof(ar));+ if (q == nullptr || out == nullptr) return false;+ if (ar.gen == 0 && T > 16) { std::fprintf(stderr, "dsv41 metal: sparse_attn gen0 T>16\n"); return false; }+ if (!ensure()) return false;++ void* hq = cpu_addr((void*) q);+ void* hw = cpu_addr((void*) win);+ void* hc = cpu_addr((void*) ckv);+ void* hs = cpu_addr((void*) sel);+ void* hk = cpu_addr((void*) sink);+ void* ho = cpu_addr((void*) out);+ if (!hq || !ho) {+ std::fprintf(stderr, "dsv41 metal: sparse_attn has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };++ int wmax = 0;+ if (ar.gen != 0) {+ wmax = ar.win + T + 1;+ } else {+ for (int t = 0; t < T; ++t) wmax = ar.win0[t] + ar.nwin[t] > wmax ? ar.win0[t] + ar.nwin[t] : wmax;+ wmax += 1;+ }+ if (wmax > 0 && hw == nullptr) { std::fprintf(stderr, "dsv41 metal: sparse_attn win null (wmax %d)\n", wmax); return false; }+ const size_t wrows = (size_t) (wmax > 0 ? wmax : 1);+ size_t crows = 0;+ if (ckv_fmt == 2) {+ if (ar.gen != 0 && ar.R > 0) {+ crows = (size_t) ((ar.p0 + ar.t0 + T + 1) / ar.R) + 2; // every selected row is below this position+ } else {+ // (the rows the selection names are on the device and not read back here: an earlier build read them on the+ // host while the kernel that wrote them was still queued, and sized the binding from stale zeros)+ const size_t rem = ds4_gpu_alloc_remaining(hc);+ crows = rem > 0 ? rem / 288 : (size_t) (ar.nsel[0] > 0 ? ar.nsel[0] : 1);+ }+ if (crows < 1) crows = 1;+ } else {+ crows = (size_t) (T > 16 ? T : 16);+ }+ const size_t qbytes = (size_t) T * (size_t) heads * 512 * 4;+ const size_t wbytes = wrows * 512 * 2;+ const size_t cbytes = crows * (ckv_fmt == 2 ? 288 : 1024);+ const size_t obytes = (size_t) T * (size_t) heads * 512 * 4;+ const size_t sbytes = (size_t) T * (size_t) (sel_stride > 0 ? sel_stride : 1) * 4;+ mtl::Buffer bq = wrap(hq, qbytes);+ mtl::Buffer bw = hw != nullptr ? wrap(hw, wbytes) : g_cx->alloc(64);+ mtl::Buffer bc = hc != nullptr ? wrap(hc, cbytes) : g_cx->alloc(64);+ mtl::Buffer bs = hs != nullptr ? wrap(hs, sbytes) : g_cx->alloc(4);+ mtl::Buffer bk = hk != nullptr ? wrap(hk, (size_t) heads * 4) : g_cx->alloc((size_t) heads * 4);+ mtl::Buffer bo = wrap(ho, obytes);+ if (!bq.handle || !bw.handle || !bc.handle || !bs.handle || !bk.handle || !bo.handle) {+ std::fprintf(stderr, "dsv41 metal: sparse_attn wrap failed\n");+ return false;+ }++ int G = 8;+ if (heads % 8 != 0) G = heads % 4 == 0 ? 4 : (heads % 2 == 0 ? 2 : 1);+ while (G > 1 && (heads / G) * T < 64) G /= 2; // keep enough threadgroups on the GPU (CUDA's 2 * SMs, in effect)++ static const int attn_mode = std::getenv("DS4_ATTN_MX") ? std::atoi(std::getenv("DS4_ATTN_MX")) : 1; // 1 (default): matrix-unit kernel; 0: the scalar one; 2: both, compare+ const bool use_mx = ckv_fmt == 2 && G == 8 && attn_mode != 0;- const char* fn = use_mx ? "ds4_sparse_attn_mx" : ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4"++ // DS4_ATTN2Q: two queries a threadgroup, sharing the staged keys (the staging is ~20 % of a 32K prompt and is per++ // key, so sharing halves it). Verified bit-exact in tools/ds4/attn_harness, including the causal shape where the++ // two rows have different window lengths.++ static const bool attn2q_env = std::getenv("DS4_ATTN2Q") != nullptr && use_mx;++ // Two queries a threadgroup need a shared window base and selection list (the kernel stages one key set for both,++ // and the causal shape gives the second query a superset only when those agree). The rows carry their own++ // win0/nwin/nsel, so the pairing is decided here, per dispatch, and never assumed.++ bool attn2q = attn2q_env;++ if (attn2q) {++ for (int t = 1; t < T; ++t) {++ if (ar.win0[t] != ar.win0[0] || ar.nsel[t] != ar.nsel[0] || ar.nwin[t] < ar.nwin[t - 1]) { attn2q = false; break; }++ }++ }++ const int qpt = attn2q ? 2 : 1;++ const char* fn = attn2q ? "ds4_sparse_attn_mx2" : use_mx ? "ds4_sparse_attn_mx" : ckv_fmt == 2 ? (G == 8 ? "ds4_sparse_attn_g8" : G == 4 ? "ds4_sparse_attn_g4"+ : G == 2 ? "ds4_sparse_attn_g2" : "ds4_sparse_attn_g1")+ : "ds4_sparse_attn_bf16";+ mtl::Pipeline pl;+ try {- pl = g_cx->pipeline(g_lib, fn);++ // DS4_ATTN_PROBE=N: the attention timing probe (see ATTN_PROBE in ds4_init.metal): 1 skips P.V, 2 skips QK.++ // The numbers it produces are wrong on purpose.++ const char* ap = std::getenv("DS4_ATTN_PROBE");++ if (ap != nullptr && std::atoi(ap) != 0) {++ mtl::Constants cs;++ cs.i(25, std::atoi(ap));++ pl = g_cx->pipeline_uncached(g_lib, fn, cs);++ } else {++ pl = g_cx->pipeline(g_lib, fn);++ }+ } catch (...) {+ std::fprintf(stderr, "dsv41 metal: sparse_attn pipeline %s unavailable\n", fn);+ return false;+ }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bw, (hw != nullptr ? off(hw) : (size_t) 0)),+ mtl::Arg::B(bc, (hc != nullptr ? off(hc) : (size_t) 0)),+ mtl::Arg::B(bs, (hs != nullptr ? off(hs) : (size_t) 0)),+ mtl::Arg::V((unsigned) (sel_stride > 0 ? sel_stride : 1)), mtl::Arg::V(ar),+ mtl::Arg::B(bk, (hk != nullptr ? off(hk) : (size_t) 0)), mtl::Arg::V((unsigned) heads), mtl::Arg::V(scale),+ mtl::Arg::B(bo, off(ho))};+ dsp.grid[0] = (uint32_t) (ckv_fmt == 2 ? heads / G : heads);- dsp.grid[1] = (uint32_t) T;++ dsp.grid[1] = (uint32_t) ((T + qpt - 1) / qpt);+ dsp.grid[2] = 1;+ dsp.tg[0] = 256;+ dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ if (use_mx && attn_mode == 2) { // debugging: the scalar kernel on the same inputs, then the difference+ mtl::Pipeline ps;+ try { ps = g_cx->pipeline(g_lib, "ds4_sparse_attn_g8"); } catch (...) { return true; }+ mtl::Buffer bo2 = g_cx->alloc(obytes);+ mtl::Dispatch d2 = dsp;+ d2.p = &ps;+ d2.args.back() = mtl::Arg::B(bo2, 0);+ g_cx->run({d2});+ g_cx->flush_runs();+ const float* a = (const float*) ho;+ const float* c = (const float*) bo2.ptr;+ double mx_abs = 0, mx_ref = 0, sum_sq = 0, sum_ref = 0;+ for (size_t i = 0; i < obytes / 4; ++i) {+ const double df = std::fabs((double) a[i] - (double) c[i]);+ mx_abs = std::max(mx_abs, df);+ mx_ref = std::max(mx_ref, (double) std::fabs(c[i]));+ sum_sq += df * df;+ sum_ref += (double) c[i] * (double) c[i];+ }+ static double worst_rel = 0; static long calls = 0;+ const double rel = std::sqrt(sum_sq / (sum_ref + 1e-30));+ worst_rel = std::max(worst_rel, rel);+ if (++calls % 50 == 1) std::fprintf(stderr, "attn mx vs scalar: call %ld T %d max abs %.3g (max |ref| %.3g) rms rel %.3g worst rms rel %.3g\n", calls, T, mx_abs, mx_ref, rel, worst_rel);+ g_cx->free(bo2);+ }+ return true;+ }+++ // ds4_kernels.cu:461 route_kernel - sqrt(softplus(logits)) top k with bias, then the unbiased weights. vl is a VlBits+ // (1024 bytes) or null; bias_vl applies to the rows its bits select.+ bool route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,+ const float* bias_vl, const void* vl) {+ if (T <= 0) return true;+ if (logits == nullptr || bias == nullptr || idx == nullptr || w == nullptr || n <= 0 || n > 1024 || k <= 0 || k > 8)+ return false;+ if (!ensure()) return false;+ void* hl = cpu_addr((void*) logits);+ void* hb = cpu_addr((void*) bias);+ void* hi = cpu_addr((void*) idx);+ void* hw = cpu_addr((void*) w);+ void* hv = bias_vl != nullptr ? cpu_addr((void*) bias_vl) : nullptr;+ if (!hl || !hb || !hi || !hw) {+ std::fprintf(stderr, "dsv41 metal: route has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4);+ mtl::Buffer bb = wrap(hb, (size_t) n * 4);+ mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4);+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4);+ mtl::Buffer bv = hv != nullptr ? wrap(hv, (size_t) n * 4) : g_cx->alloc(4);+ mtl::Buffer bbv = g_cx->alloc(1024);+ if (!bl.handle || !bb.handle || !bi.handle || !bw.handle || !bv.handle || !bbv.ptr) return false;+ const bool has_vl = vl != nullptr;+ if (has_vl) std::memcpy(bbv.ptr, vl, 1024);+ mtl::Pipeline pl;+ try {+ pl = g_cx->pipeline(g_lib, "ds4_route");+ } catch (...) {+ std::fprintf(stderr, "dsv41 metal: route pipeline unavailable\n");+ return false;+ }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),+ mtl::Arg::V(route_scale), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),+ mtl::Arg::B(bv, hv != nullptr ? off(hv) : 0), mtl::Arg::V(has_vl ? 1u : 0u), mtl::Arg::B(bbv)};+ dsp.grid[0] = (uint32_t) T;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 512;+ dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ g_cx->free(bbv);+ return true;+ }+++ namespace {+ struct Ds4GatherJob {+ int src[16 * 8];+ };+ struct Ds4CombineJob {+ int n[16];+ int row[16][8];+ float w[16][8];+ int cpu[16];+ };+ static_assert(sizeof(Ds4GatherJob) == 512, "Ds4GatherJob layout must match the MSL struct");+ static_assert(sizeof(Ds4CombineJob) == 1152, "Ds4CombineJob layout must match the MSL struct");+ } // namespace++ // ds4_kernels.cu:1336 gather_kernel - out[r] = x[g.src[r]] for n rows of cols floats+ bool gather_rows(const float* x, const void* g, int n, int cols, float* out) {+ if (n <= 0 || cols <= 0) return true;+ if (x == nullptr || g == nullptr || out == nullptr || n > 16 * 8) return false;+ if (!ensure()) return false;+ void* hx = cpu_addr((void*) x);+ void* ho = cpu_addr((void*) out);+ if (!hx || !ho) return false;+ Ds4GatherJob gj{};+ std::memcpy(&gj, g, sizeof(gj));+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ int max_src = 0;+ for (int i = 0; i < n; ++i) max_src = gj.src[i] > max_src ? gj.src[i] : max_src;+ mtl::Buffer bx = wrap(hx, (size_t) (max_src + 1) * cols * 4); // src rows may be beyond the n output rows+ mtl::Buffer bo = wrap(ho, (size_t) n * cols * 4);+ if (!bx.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_gather"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V(gj), mtl::Arg::V((unsigned) cols), mtl::Arg::B(bo, off(ho))};+ dsp.grid[0] = (uint32_t) ((cols + 255) / 256);+ dsp.grid[1] = (uint32_t) n;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }++ // ds4_kernels.cu:1342 combine_kernel - out[t] = sum_j w[t][j] ed[row[t][j]] + cpu[t] (when set) + sh[t]+ bool combine(const float* ed, const float* cpu, const float* sh, const void* c, int T, int dim, float* out) {+ if (T <= 0 || dim <= 0) return true;+ if (ed == nullptr || c == nullptr || out == nullptr || T > 16) return false;+ if (!ensure()) return false;+ void* he = cpu_addr((void*) ed);+ void* hc = cpu != nullptr ? cpu_addr((void*) cpu) : nullptr;+ void* hs = sh != nullptr ? cpu_addr((void*) sh) : nullptr;+ void* ho = cpu_addr((void*) out);+ if (!he || !ho) return false;+ Ds4CombineJob cj{};+ std::memcpy(&cj, c, sizeof(cj));+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ int max_row = 0;+ for (int t = 0; t < T; ++t)+ for (int j = 0; j < cj.n[t]; ++j) max_row = cj.row[t][j] > max_row ? cj.row[t][j] : max_row;+ mtl::Buffer be = wrap(he, (size_t) (max_row + 1) * dim * 4); // rows may be beyond T * kPicks+ mtl::Buffer bc = hc != nullptr ? wrap(hc, (size_t) T * dim * 4) : g_cx->alloc(4);+ mtl::Buffer bs = hs != nullptr ? wrap(hs, (size_t) T * dim * 4) : g_cx->alloc(4);+ mtl::Buffer bo = wrap(ho, (size_t) T * dim * 4);+ if (!be.handle || !bc.handle || !bs.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_combine"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(be, off(he)), mtl::Arg::B(bc, hc != nullptr ? off(hc) : 0),+ mtl::Arg::B(bs, hs != nullptr ? off(hs) : 0), mtl::Arg::V(cj), mtl::Arg::V((unsigned) dim),+ mtl::Arg::V(hc != nullptr ? 1u : 0u), mtl::Arg::V(hs != nullptr ? 1u : 0u), mtl::Arg::B(bo, off(ho))};+ dsp.grid[0] = (uint32_t) ((dim + 255) / 256);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }+++ // On Apple silicon every device pointer is a CPU address (unified memory), so a pointer the gpu allocator does not know about+ // (the engine builds some of its small arrays itself) still wraps correctly: host_view when it has one, the pointer otherwise.+ static inline void* cpu_addr(void* p) {+ void* h = strata::gpu::host_view(p);+ return h ? h : p;+ }++ // ds4_kernels.cu:1286 engram_dequant_kernel - the Engram rows (fmt 2 = IQ4_NL, the tables this model's pack carries; the+ // other formats are left to the CUDA launch, which reports the line)+ bool engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, int fmt) {+ (void) scales; // (fmt 2 keeps every block's scale in its own 18 bytes)+ if (n <= 0 || hd <= 0) return true;+ if (fmt != 2 || rows == nullptr || out == nullptr) return false;+ if (!ensure()) return false;+ void* hr = cpu_addr((void*) rows);+ void* ho = cpu_addr((void*) out);+ if (!hr || !ho) { std::fprintf(stderr, "dsv41 metal: engram_dequant has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ // (the kernel strides rows by hd bytes, as the CUDA kernel does, though an IQ4_NL row only fills hd/32*18 of them:+ // wrapping n * 18/32 * hd left the last rows outside the buffer - garbage scales, measured as inf at row 257 of 288)+ mtl::Buffer br = wrap(hr, (size_t) n * (size_t) hd);+ mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) hd * 4);+ if (!br.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_dequant_rows"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(br, off(hr)), mtl::Arg::V((unsigned) hd), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(fmt)};+ d.grid[0] = (uint32_t) n;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }++ // ds4_kernels.cu:1311 engram_gate_kernel - the Engram gate over a (token, copy) row: h += sigmoid(normalised dot) * val+ bool engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps) {+ if (T <= 0 || hc <= 0 || dim <= 0) return true;+ if (h == nullptr || kv == nullptr || w == nullptr) return false;+ if (!ensure()) return false;+ void* hh = cpu_addr((void*) h);+ void* hk = cpu_addr((void*) kv);+ void* hw = cpu_addr((void*) w);+ if (!hh || !hk || !hw) { std::fprintf(stderr, "dsv41 metal: engram_gate has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);+ mtl::Buffer bk = wrap(hk, (size_t) T * (size_t) (hc + 1) * (size_t) dim * 4);+ mtl::Buffer bw = wrap(hw, (size_t) hc * (size_t) dim * 4);+ if (!bh.handle || !bk.handle || !bw.handle) {+ std::fprintf(stderr, "dsv41 metal: engram_gate wrap failed (h %p kv %p w %p; %p %p %p)\n", hh, hk, hw,+ bh.handle, bk.handle, bw.handle);+ return false;+ }+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_engram_gate"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bk, off(hk)), mtl::Arg::B(bw, off(hw)),+ mtl::Arg::V((unsigned) hc), mtl::Arg::V((unsigned) dim), mtl::Arg::V(eps)};+ d.grid[0] = (uint32_t) hc;+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }++ namespace {+ // mirrors metal/src/engine/msl/ds4_init.metal's Ds4Rows (ds4_kernels.cuh:115 Rows, kWin = 16)+ struct Ds4RowsJob {+ int n[16];+ int k[16];+ int gen, R, kcap, ccap, blk;+ int64_t p0;+ };+ } // namespace++ // ds4_kernels.cu:524 compress_kernel - the window compressor (ratio > 1): each token's kv / score row into its slot, and+ // every ratio'th token pooled by a per-value softmax over the slots, scaled by the row RMS and the norm weights+ bool compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,+ int d, const float* w, float eps, float* latent) {+ if (T <= 0 || d <= 0 || ratio <= 0) return true;+ if (d > 2 * 512) return false; // (the kernel holds d <= 2 * blockDim)+ if (kv == nullptr || score == nullptr || kv_state == nullptr || score_state == nullptr || w == nullptr ||+ latent == nullptr)+ return false;+ if (!ensure()) return false;+ void* hkv = cpu_addr((void*) kv);+ void* hsc = cpu_addr((void*) score);+ void* hks = cpu_addr((void*) kv_state);+ void* hss = cpu_addr((void*) score_state);+ void* hw = cpu_addr((void*) w);+ void* hla = cpu_addr((void*) latent);+ if (!hkv || !hsc || !hks || !hss || !hw || !hla) {+ std::fprintf(stderr, "dsv41 metal: compress_win has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bkv = wrap(hkv, (size_t) T * (size_t) d * 4);+ mtl::Buffer bsc = wrap(hsc, (size_t) T * (size_t) d * 4);+ mtl::Buffer bks = wrap(hks, (size_t) ratio * (size_t) d * 4);+ mtl::Buffer bss = wrap(hss, (size_t) ratio * (size_t) d * 4);+ mtl::Buffer bw = wrap(hw, (size_t) d * 4);+ mtl::Buffer bla = wrap(hla, (size_t) T * (size_t) d * 4);+ if (!bkv.handle || !bsc.handle || !bks.handle || !bss.handle || !bw.handle || !bla.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_compress"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bkv, off(hkv)), mtl::Arg::B(bsc, off(hsc)), mtl::Arg::B(bks, off(hks)),+ mtl::Arg::B(bss, off(hss)), mtl::Arg::V(ratio), mtl::Arg::V(p0), mtl::Arg::V(T),+ mtl::Arg::V((unsigned) d), mtl::Arg::B(bw, off(hw)), mtl::Arg::V(eps), mtl::Arg::B(bla, off(hla))};+ dsp.grid[0] = 1;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 512; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }++ // ds4_kernels.cu:582 index_score_kernel<128, kfmt> - the indexer's scores: a position's key row against every head's+ // query, relu, the heads' weighted sum (rows = ds4_kernels.cuh's Rows, by value there and copied here)+ bool index_score(const float* q, const float* wq, const void* kc, int heads, int hd, int T, int n, const int* cand,+ int cand_stride, int blk, const void* rows, float* out, long long out_stride, int kfmt) {+ if (n <= 0 || T <= 0) return true;+ if (hd != 128 || heads <= 0 || heads > 32 || (kfmt != 3 && kfmt != 0)) return false;+ if (q == nullptr || wq == nullptr || kc == nullptr || out == nullptr || rows == nullptr) return false;+ if (!ensure()) return false;+ void* hq = cpu_addr((void*) q);+ void* hwq = cpu_addr((void*) wq);+ void* hkc = cpu_addr((void*) kc);+ void* hout = cpu_addr((void*) out);+ if (!hq || !hwq || !hkc || !hout) {+ std::fprintf(stderr, "dsv41 metal: index_score has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ const size_t kb = kfmt == 3 ? (size_t) ((hd / 2 + hd / 32 + 15) / 16 * 16) : (size_t) hd * 2;+ mtl::Buffer bq = wrap(hq, (size_t) T * (size_t) heads * (size_t) hd * 4);+ mtl::Buffer bwq = wrap(hwq, (size_t) T * 64 * 4);+ mtl::Buffer bkc = wrap(hkc, (size_t) n * kb);+ mtl::Buffer bo = wrap(hout, ((size_t) (T - 1) * (size_t) out_stride + (size_t) n) * 4);+ if (!bq.handle || !bwq.handle || !bkc.handle || !bo.handle) return false;+ const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;+ mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);+ if (!bc.handle) return false;+ // Rows is a host struct: it travels as a copy (no gpu address)+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: index_score rows buffer failed\n"); return false; }+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_index_score"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bwq, off(hwq)), mtl::Arg::B(bkc, off(hkc)),+ mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::B(bo, off(hout)),+ mtl::Arg::B(br), mtl::Arg::V(heads), mtl::Arg::V(n), mtl::Arg::V(cand_stride), mtl::Arg::V(blk),+ mtl::Arg::V((long) out_stride), mtl::Arg::V(kfmt), mtl::Arg::V(cand != nullptr ? 1 : 0)};++ // DS4_IDX_MMA: the matrix-unit indexer (ds4_index_score_mma). It stages 16 fp4 keys a threadgroup and runs the++ // 32 x 128 x 16 head-dot product through simdgroup_matrix - 3.17-6.10x the scalar loop in the standalone harness,++ // with every output matching to half precision. Opt-in until the gate and KL say the rounding is acceptable.++ // On by default: measured +7.7 to +12 % on the 32K prompt and +2 % at 3K, with the KL unchanged on both packs++ // (0.11018 and 0.17087). DS4_NO_IDX_MMA=1 restores the scalar indexer; on the 2.00 pack the score hash moves++ // (e81282a3 -> 0800179d) because half rounding breaks token ties, while KL and top-1 are identical.++ static const bool idx_mma = std::getenv("DS4_NO_IDX_MMA") == nullptr;++ if (idx_mma && cand == nullptr && kfmt == 3) {++ mtl::Pipeline pm;++ try { pm = g_cx->pipeline(g_lib, "ds4_index_score_mma"); } catch (...) { pm = mtl::Pipeline{}; }++ if (pm.handle) {++ mtl::Dispatch dm;++ dm.p = ±++ dm.args = {mtl::Arg::B(bq, off(hq)), mtl::Arg::B(bwq, off(hwq)), mtl::Arg::B(bkc, off(hkc)),++ mtl::Arg::B(bc, 0), mtl::Arg::B(bo, off(hout)), mtl::Arg::B(br),++ mtl::Arg::V(heads), mtl::Arg::V(n), mtl::Arg::V((int) T), mtl::Arg::V((long) out_stride)};++ dm.grid[0] = (uint32_t) ((n + 15) / 16);++ dm.grid[1] = (uint32_t) T;++ dm.grid[2] = 1;++ dm.tg[0] = 256; dm.tg[1] = dm.tg[2] = 1;++ g_cx->run({dm});++ g_cx->free(br);++ if (bc.handle && cand != nullptr) g_cx->free(bc);++ return true;++ }++ }+ dsp.grid[0] = (uint32_t) ((n + 255) / 256);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ g_cx->free(br);+ if (cand == nullptr) g_cx->free(bc);+ return true;+ }++ // ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations, a warp a row of the matrix and 2 tokens+ // a block (the indexer's wproj: ds4_model.cu:7743)+ bool gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy) {+ if (T <= 0 || n <= 0 || k <= 0) return true;+ if (k % 8 != 0 || W == nullptr || x == nullptr || y == nullptr) return false;+ if (!ensure()) return false;+ void* hw = cpu_addr((void*) W);+ void* hx = cpu_addr((void*) x);+ void* hy = cpu_addr((void*) y);+ if (!hw || !hx || !hy) {+ std::fprintf(stderr, "dsv41 metal: gemv_f16_rows has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bw = wrap(hw, (size_t) n * (size_t) k * 2);+ mtl::Buffer bx = wrap(hx, ((size_t) (T - 1) * ldx + (size_t) k) * 4);+ mtl::Buffer by = wrap(hy, ((size_t) (T - 1) * ldy + (size_t) n) * 4);+ if (!bw.handle || !bx.handle || !by.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f16_rows"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V(n),+ mtl::Arg::V(k), mtl::Arg::V(T), mtl::Arg::V(alpha), mtl::Arg::B(by, off(hy)),+ mtl::Arg::V((unsigned long) ldy)};+ dsp.grid[0] = (uint32_t) ((n + 7) / 8);+ dsp.grid[1] = (uint32_t) ((T + 1) / 2);+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }++ // ds4_kernels.cu:650 topk_kernel - a token's k largest indexer scores (four 8-bit radix passes, then the picks in index+ // order); Rows by value there and copied here+ bool topk(const float* vals, long long stride, int T, const void* rows, const int* cand, int cand_stride, int blk, int* out,+ int out_stride) {+ if (T <= 0) return true;+ if (vals == nullptr || out == nullptr || rows == nullptr) return false;+ if (!ensure()) return false;+ void* hv = cpu_addr((void*) vals);+ void* ho = cpu_addr((void*) out);+ if (!hv || !ho) {+ std::fprintf(stderr, "dsv41 metal: topk has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bv = wrap(hv, ((size_t) (T - 1) * (size_t) stride + 1) * 4);+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + 1) * 4);+ if (!bv.handle || !bo.handle) return false;+ const size_t cbytes = (size_t) T * (size_t) (cand_stride > 0 ? cand_stride : 1) * 4;+ mtl::Buffer bc = cand != nullptr ? wrap(cpu_addr((void*) cand), cbytes) : g_cx->alloc(4);+ if (!bc.handle) return false;+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: topk rows buffer failed\n"); return false; }+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_topk"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bv, off(hv)), mtl::Arg::V((long) stride), mtl::Arg::B(br),+ mtl::Arg::B(bc, cand != nullptr ? off(cpu_addr((void*) cand)) : 0), mtl::Arg::V(cand_stride),+ mtl::Arg::V(blk), mtl::Arg::B(bo, off(ho)), mtl::Arg::V(out_stride),+ mtl::Arg::V(cand != nullptr ? 1 : 0)};+ dsp.grid[0] = (uint32_t) T;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ g_cx->free(br);+ if (cand == nullptr) g_cx->free(bc);+ return true;+ }++ // ds4_kernels.cu:751 block_max_kernel - a block of the indexer's positions and its largest score (the newest pinned at +inf)+ bool block_max(const float* score, long long stride, int T, const void* rows, int blk, float* bm, long long bm_stride,+ int nb_max) {+ if (T <= 0 || nb_max <= 0 || blk <= 0) return true;+ if (score == nullptr || bm == nullptr || rows == nullptr) return false;+ if (!ensure()) return false;+ void* hs = cpu_addr((void*) score);+ void* hb = cpu_addr((void*) bm);+ if (!hs || !hb) {+ std::fprintf(stderr, "dsv41 metal: block_max has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bs = wrap(hs, ((size_t) (T - 1) * (size_t) stride + (size_t) nb_max * (size_t) blk) * 4);+ mtl::Buffer bb = wrap(hb, ((size_t) (T - 1) * (size_t) bm_stride + (size_t) nb_max) * 4);+ if (!bs.handle || !bb.handle) return false;+ mtl::Buffer br = g_cx->alloc(sizeof(Ds4RowsJob));+ if (!br.ptr) { std::fprintf(stderr, "dsv41 metal: block_max rows buffer failed\n"); return false; }+ std::memcpy(br.ptr, rows, sizeof(Ds4RowsJob));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_block_max"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::V((long) stride), mtl::Arg::B(br), mtl::Arg::V(blk),+ mtl::Arg::B(bb, off(hb)), mtl::Arg::V((long) bm_stride)};+ dsp.grid[0] = (uint32_t) ((nb_max + 255) / 256);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ g_cx->free(br);+ return true;+ }++ // ds4_kernels.cu:1141 hc_mean_kernel - a token's mean over its hc streams (grid (ceil(dim / 256), T), 256 threads)+ bool hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride) {+ if (T <= 0 || hc <= 0 || dim <= 0) return true;+ if (h == nullptr || out == nullptr) return false;+ if (!ensure()) return false;+ void* hh = cpu_addr((void*) h);+ void* ho = cpu_addr((void*) out);+ if (!hh || !ho) {+ std::fprintf(stderr, "dsv41 metal: hc_mean has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * (size_t) hc * (size_t) dim * 4);+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) out_stride + (size_t) dim) * 4);+ if (!bh.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mean"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::V(hc), mtl::Arg::V(dim), mtl::Arg::B(bo, off(ho)),+ mtl::Arg::V((long) out_stride)};+ dsp.grid[0] = (uint32_t) ((dim + 255) / 256);+ dsp.grid[1] = (uint32_t) T;+ dsp.grid[2] = 1;+ dsp.tg[0] = 256; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }++ // ds4_kernels.cu:1150 iota_rows_kernel - out[t][i] = i+ // ds4_kernels.cu:1156 markov_kernel / :1265 confidence_kernel - the drafter's head. markov_add is a warp a vocabulary+ // row (the previous token's E row staged in the threadgroup, then E . H[v] added to logits[v]); confidence a threadgroup+ // a token. Both jobs are the kernels' own structs (metal/src/engine/msl/ds4_init.metal's Ds4MarkovJob / Ds4ConfJob).+ namespace {+ // (not an entry point: the generator takes only the ^bool lines) mirrors ds4_init.metal's Ds4MarkovJob / Ds4ConfJob+ struct MarkovJob {+ void* logits;+ const void* E;+ const void* H;+ const void* tok;+ int V, R;+ };+ static_assert(sizeof(MarkovJob) == 40, "Ds4MarkovJob's layout");++ struct ConfJob {+ const void* x;+ const void* E;+ const void* tok;+ const void* w;+ void* out;+ int dim, R;+ };+ static_assert(sizeof(ConfJob) == 48, "Ds4ConfJob's layout");+ } // namespace++ bool markov_add(const float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R) {+ if (logits == nullptr || E == nullptr || H == nullptr || tok == nullptr) return false;+ if (V <= 0 || R <= 0 || R > 1024) return false; // (the kernel's staged E row holds 1024 floats)+ if (!ensure()) return false;+ MarkovJob j{};+ j.logits = const_cast<float*>(logits);+ j.E = E;+ j.H = H;+ j.tok = tok;+ j.V = V;+ j.R = R;+ mtl::Buffer bj = g_cx->alloc(sizeof(j));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: markov_add job buffer failed\n");+ return false;+ }+ std::memcpy(bj.ptr, &j, sizeof(j));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_markov_add"); } catch (...) { g_cx->free(bj); return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) ((V + 7) / 8);+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+ }++ bool confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out) {+ if (x == nullptr || E == nullptr || tok == nullptr || w == nullptr || out == nullptr) return false;+ if (T <= 0 || dim <= 0 || R <= 0) return false;+ if (!ensure()) return false;+ ConfJob j{};+ j.x = x;+ j.E = E;+ j.tok = tok;+ j.w = w;+ j.out = out;+ j.dim = dim;+ j.R = R;+ mtl::Buffer bj = g_cx->alloc(sizeof(j));+ if (!bj.ptr) {+ std::fprintf(stderr, "dsv41 metal: confidence job buffer failed\n");+ return false;+ }+ std::memcpy(bj.ptr, &j, sizeof(j));+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_confidence"); } catch (...) { g_cx->free(bj); return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bj)};+ d.grid[0] = (uint32_t) T;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 1024; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bj);+ return true;+ }++ bool iota_rows(int* out, int T, int n, int stride) {+ if (T <= 0 || n <= 0) return true;+ if (out == nullptr) return false;+ if (!ensure()) return false;+ void* ho = cpu_addr((void*) out);+ if (!ho) {+ std::fprintf(stderr, "dsv41 metal: iota_rows has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bo = wrap(ho, ((size_t) (T - 1) * (size_t) stride + (size_t) n) * 4);+ if (!bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_iota_rows"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bo, off(ho)), mtl::Arg::V(n), mtl::Arg::V(stride)};+ dsp.grid[0] = (uint32_t) T;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 64; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ return true;+ }++ // vendor/sampler.cu:93 sampler_greedy_kernel - the plain argmax over the vocabulary, one block (1024) a token, the+ // repetition penalties from the history window's bitmap (the caller takes this path when greedy or temperature <= 0)+ bool sample_greedy(const float* logits, int n_tokens, int n_vocab, const int* history, int history_len, int penalty_last_n,+ float penalty_repeat, float penalty_freq, float penalty_present, const float* logit_bias, int* out) {+ if (n_tokens <= 0 || n_vocab <= 0) return true;+ if (logits == nullptr || out == nullptr) return false;+ if ((n_vocab + 31) / 32 > 4096) return false; // (the kernel's threadgroup bitmap)+ if (!ensure()) return false;+ void* hl = cpu_addr((void*) logits);+ void* ho = cpu_addr((void*) out);+ if (!hl || !ho) {+ std::fprintf(stderr, "dsv41 metal: sample_greedy has no host view\n");+ return false;+ }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bl = wrap(hl, (size_t) n_tokens * (size_t) n_vocab * 4);+ mtl::Buffer bo = wrap(ho, (size_t) n_tokens * 4);+ if (!bl.handle || !bo.handle) return false;+ const size_t hbytes = (size_t) n_tokens * (size_t) (history_len > 0 ? history_len : 1) * 4;+ mtl::Buffer bh = history != nullptr ? wrap(cpu_addr((void*) history), hbytes) : g_cx->alloc(4);+ mtl::Buffer bb = logit_bias != nullptr ? wrap(cpu_addr((void*) logit_bias), (size_t) n_vocab * 4) : g_cx->alloc(4);+ if (!bh.handle || !bb.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_sampler_greedy"); } catch (...) { return false; }+ mtl::Dispatch dsp;+ dsp.p = &pl;+ dsp.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::V(n_vocab),+ mtl::Arg::B(bh, history != nullptr ? off(cpu_addr((void*) history)) : 0),+ mtl::Arg::V(history_len), mtl::Arg::V(penalty_last_n), mtl::Arg::V(penalty_repeat),+ mtl::Arg::V(penalty_freq), mtl::Arg::V(penalty_present),+ mtl::Arg::B(bb, logit_bias != nullptr ? off(cpu_addr((void*) logit_bias)) : 0),+ mtl::Arg::V(logit_bias != nullptr ? 1 : 0), mtl::Arg::B(bo, off(ho))};+ dsp.grid[0] = (uint32_t) n_tokens;+ dsp.grid[1] = dsp.grid[2] = 1;+ dsp.tg[0] = 1024; dsp.tg[1] = dsp.tg[2] = 1;+ g_cx->run({dsp});+ if (history == nullptr) g_cx->free(bh);+ if (logit_bias == nullptr) g_cx->free(bb);+ return true;+ }++ bool unpack_sel(const void* ckv, const int* sel, int sel_stride, int gen, long long p0, int t0, int R, int kcap,+ const int* nsel, int cap, void* out, int T, int nsel_count, size_t row_bytes) {+ if (T <= 0 || sel_stride <= 0 || nsel_count <= 0) return true;+ // nothing selected yet (a prompt at position 0): the scratch stays untouched, which is what the CUDA launch would do with+ // no rows to write+ if (ckv == nullptr || sel == nullptr || out == nullptr) return true;+ if (!ensure()) return false;+ void* hc = cpu_addr((void*) ckv);+ void* hs = cpu_addr((void*) sel);+ void* ho = cpu_addr((void*) out);+ if (!hc || !hs || !ho) { std::fprintf(stderr, "dsv41 metal: unpack_sel has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ int maxrow = -1;+ if (gen != 0 && R > 0) {+ maxrow = (int) ((p0 + t0 + T) / R) + 1;+ } else if (gen == 0 && nsel != nullptr) {+ for (int t = 0; t < T && t < nsel_count; ++t)+ for (int i = 0; i < nsel[t]; ++i) {+ const int r = sel[(size_t) t * (size_t) sel_stride + (size_t) i];+ if (r > maxrow) maxrow = r;+ }+ }+ mtl::Buffer bc = wrap(hc, (size_t) (maxrow + 1 > 0 ? maxrow + 1 : 1) * row_bytes);+ mtl::Buffer bsl = wrap(hs, (size_t) T * sel_stride * 4 );+ mtl::Buffer bo = wrap(ho, (size_t) T * cap * 512 * 2 );+ if (!bc.handle || !bsl.handle || !bo.handle) return false;+ mtl::Buffer bn = g_cx->alloc((size_t) nsel_count * 4);+ if (!bn.ptr) return false;+ std::memcpy(bn.ptr, nsel, (size_t) nsel_count * 4);+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_unpack_sel"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bc, off(hc)), mtl::Arg::B(bsl, off(hs)), mtl::Arg::V((unsigned) sel_stride),+ mtl::Arg::V((int) gen), mtl::Arg::V((long) p0), mtl::Arg::V((int) t0), mtl::Arg::V((int) R),+ mtl::Arg::V((int) kcap), mtl::Arg::B(bn), mtl::Arg::V((unsigned) cap), mtl::Arg::B(bo, off(ho)),+ mtl::Arg::V((unsigned long) row_bytes)};+ d.grid[0] = (uint32_t) ((sel_stride + 15) / 16);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bn);+ return true;+ }+++ // ds4_kernels.cu:1772 - the expert outputs back into the token rows+ bool moe_combine(void* acc, const void* y, const int* pos, const float* w_, int T, int k, int r0, int r1, int dim) {+ if (T <= 0 || dim <= 0) return true;+ if (!ensure()) return false;+ void* ha = cpu_addr(acc);+ void* hy = cpu_addr((void*) y);+ void* hp = cpu_addr((void*) pos);+ void* hw = cpu_addr((void*) w_);+ if (!ha || !hy || !hp || !hw) { std::fprintf(stderr, "dsv41 metal: moe_combine has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer ba = wrap(ha, (size_t) T * dim * 4 );+ mtl::Buffer by = wrap(hy, (size_t) (r1 > r0 ? r1 - r0 : 1) * dim * 4 );+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );+ if (!ba.handle || !by.handle || !bp.handle || !bw.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_moe_combine"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(ba, off(ha)), mtl::Arg::B(by, off(hy)), mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bw, off(hw)),+ mtl::Arg::V((int) k), mtl::Arg::V((int) r0), mtl::Arg::V((int) r1), mtl::Arg::V((unsigned) dim)};+ d.grid[0] = (uint32_t) ((dim + 255) / 256);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }+++ // ds4_kernels.cu:362 - hc_post: out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]+ bool hc_post_v(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T, float* out) {+ if (T <= 0 || dim <= 0) return true;+ if (!ensure()) return false;+ void* hy = cpu_addr((void*) y);+ void* hr = cpu_addr((void*) res);+ void* hp = cpu_addr((void*) post);+ void* hco = cpu_addr((void*) comb);+ void* ho = cpu_addr((void*) out);+ if (!hy || !hr || !hp || !hco || !ho) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 );+ mtl::Buffer br = wrap(hr, (size_t) T * hc * dim * 4 );+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 4 );+ mtl::Buffer bc = wrap(hco, (size_t) T * 64 * 4 );+ mtl::Buffer bo = wrap(ho, (size_t) T * hc * dim * 4 );+ if (!by.handle || !br.handle || !bp.handle || !bc.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_post_v"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp)),+ mtl::Arg::B(bc, off(hco)), mtl::Arg::B(bo, off(ho)), mtl::Arg::V((unsigned) dim),+ mtl::Arg::V((unsigned) hc)};+ d.grid[0] = (uint32_t) ((dim * hc + 255) / 256);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }+++ // ds4_kernels.cu:441 - the F32 GEMV (residual / dense projections)+ bool gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy) {+ if (T <= 0 || n <= 0 || k <= 0) return true;+ if (!ensure()) return false;+ void* hw = cpu_addr((void*) W);+ void* hx = cpu_addr((void*) x);+ void* hy = cpu_addr((void*) y);+ if (!hw || !hx || !hy) { std::fprintf(stderr, "dsv41 metal: gemv_f32 has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bw = wrap(hw, (size_t) n * k * 4 );+ mtl::Buffer bx = wrap(hx, (size_t) T * k * 4 );+ mtl::Buffer by = wrap(hy, (size_t) T * (ldy > 0 ? ldy : n) * 4 );+ if (!bw.handle || !bx.handle || !by.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_gemv_f32"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned) n),+ mtl::Arg::V((unsigned) k), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned) ldy)};+ d.grid[0] = (uint32_t) ((n + 7) / 8);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }+++ // ds4_kernels.cu:1353 - the router's selection (idx, w, plan_ptr). The prefetch requests stay unpublished for now.+ bool route_publish(const float* logits, const float* bias, int n, int k, int T, float route_scale, const void* vram,+ int* idx, float* w, void* plan_ptr, const float* bias_vl, unsigned vl) {+ if (T <= 0 || n <= 0) return true;+ if (!ensure()) return false;+ void* hl = cpu_addr((void*) logits);+ void* hb = cpu_addr((void*) bias);+ void* hv = cpu_addr((void*) vram);+ void* hi = cpu_addr((void*) idx);+ void* hw = cpu_addr((void*) w);+ void* hp = cpu_addr((void*) plan_ptr);+ if (!hl || !hb || !hv || !hi || !hw || !hp) { std::fprintf(stderr, "dsv41 metal: route_publish has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) { // the pages the buffer itself spans (see q8_quant_rows' note)+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bl = wrap(hl, (size_t) T * 1024 * 4 );+ mtl::Buffer bb = wrap(hb, (size_t) n * 4 );+ mtl::Buffer bv = wrap(hv, (size_t) n * 8 );+ mtl::Buffer bi = wrap(hi, (size_t) T * 8 * 4 );+ mtl::Buffer bw = wrap(hw, (size_t) T * 8 * 4 );+ mtl::Buffer bp = wrap(hp, (size_t) T * 8 * 8 );+ if (!bl.handle || !bb.handle || !bv.handle || !bi.handle || !bw.handle || !bp.handle) return false;+ mtl::Buffer bvl = bb;+ size_t vl_off = 0;+ if (bias_vl) {+ void* hvl = cpu_addr((void*) bias_vl);+ if (!hvl) return false;+ bvl = wrap(hvl, (size_t) n * 4 );+ vl_off = off(hvl);+ }+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_route_publish"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bl, off(hl)), mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n), mtl::Arg::V((unsigned) k),+ mtl::Arg::V(route_scale), mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bi, off(hi)), mtl::Arg::B(bw, off(hw)),+ mtl::Arg::B(bp, off(hp)), mtl::Arg::B(bvl, vl_off), mtl::Arg::V((unsigned) vl),+ mtl::Arg::V((unsigned) (bias_vl ? 1 : 0))};+ d.grid[0] = (uint32_t) T;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 512; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }+++ // ds4_kernels.cu swiglu: h = silu(g) * u (clamped when lim > 0)+ bool swiglu(const float* g, const float* u, int n, float lim, float* h) {+ if (n <= 0) return true;+ if (!ensure()) return false;+ void* hg = cpu_addr((void*) g);+ void* hu = cpu_addr((void*) u);+ void* hh = cpu_addr((void*) h);+ if (!hg || !hu || !hh) { std::fprintf(stderr, "dsv41 metal: swiglu has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + (size_t) n * 4;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bg = wrap(hg), bu = wrap(hu), bh = wrap(hh);+ if (!bg.handle || !bu.handle || !bh.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_swiglu"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bg, off(hg)), mtl::Arg::B(bu, off(hu)), mtl::Arg::V((unsigned) n), mtl::Arg::V(lim),+ mtl::Arg::B(bh, off(hh))};+ d.grid[0] = (uint32_t) ((n + 255) / 256);+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }+++ // vendor/moe_win.cu:49 moe_win_group - the expert -> group index for a window. One thread in CUDA; on the Mac every device+ // pointer is a CPU address, so the same 25-line loop runs on the host, which keeps the first-appearance order the kernel has and+ // avoids mirroring the 17-field MoeWin struct through a constant buffer.+ bool moe_win_group_cpu(const void* plan_ptr, void* grp_ptr, void* grp_start, void* grp_ent, void* n_groups, int T, int k) {+ if (T <= 0 || k <= 0) return true;+ if (T * 8 > 4096) { std::fprintf(stderr, "dsv41 metal: moe_win_group window of %d too wide\n", T); return false; }+ const unsigned long long* plan = (const unsigned long long*) cpu_addr((void*) plan_ptr);+ unsigned long long* gptr = (unsigned long long*) cpu_addr(grp_ptr);+ int* gstart = (int*) cpu_addr(grp_start);+ int* gent = (int*) cpu_addr(grp_ent);+ int* ng_out = (int*) cpu_addr(n_groups);+ if (!plan || !gptr || !gstart || !gent || !ng_out) return false;+ static unsigned long long gp[4096];+ static int cnt[4096], fill[4097], gof[4096];+ int ng = 0;+ for (int t = 0; t < T; ++t) {+ for (int i = 0; i < 8; ++i) {+ const int e = t * 8 + i;+ gof[e] = -1;+ const unsigned long long p = i < k ? plan[e] : 0ull;+ if (p == 0ull) continue;+ int g = 0;+ while (g < ng && gp[g] != p) ++g;+ if (g == ng) {+ gp[ng] = p;+ cnt[ng++] = 0;+ }+ ++cnt[g];+ gof[e] = g;+ }+ }+ fill[0] = 0;+ for (int g = 0; g < ng; ++g) {+ gptr[g] = gp[g];+ gstart[g] = fill[g];+ fill[g + 1] = fill[g] + cnt[g];+ }+ gstart[ng] = fill[ng];+ for (int e = 0; e < T * 8; ++e) {+ if (gof[e] >= 0) gent[fill[gof[e]]++] = e;+ }+ *ng_out = ng;+ return true;+ }+++++ // ---- the prompt chunk's dense GEMM: float rows -> half, a quantized matrix -> half rows, MPP matmul2d (kernels/dense_mm.metal)+ namespace {+ mtl::Buffer g_gm_xs, g_gm_yt; // grow-only scratch: packed activations, the contiguous result+ int g_lib_mm = -2;+ mtl::Buffer scratch(mtl::Buffer& b, size_t bytes) {+ if (!b.handle || b.bytes < bytes) {+ if (b.handle) g_cx->free(b);+ b = g_cx->alloc(bytes + (bytes >> 2));+ }+ return b;+ }+ } // namespace++ bool rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out) {+ if (nrows <= 0 || ncols <= 0) return true;+ if (!ensure()) return false;+ void* hx = strata::gpu::host_view((void*) x);+ void* ho = strata::gpu::host_view(out);+ void* hi = idx ? strata::gpu::host_view((void*) idx) : nullptr;+ if (!hx || !ho) return false;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bx = wrap(hx, ((size_t) (nrows - 1) * (size_t) ldx + (size_t) ncols) * 4);+ mtl::Buffer bo = wrap(ho, (size_t) nrows * (size_t) ncols * 2);+ mtl::Buffer bi = hi ? wrap(hi, (size_t) nrows * 4) : bx;+ if (!bx.handle || !bo.handle || !bi.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_rows_f16"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bi, hi ? off(hi) : off(hx)),+ mtl::Arg::V((unsigned) (hi ? 1 : 0)), mtl::Arg::V((unsigned) ncols), mtl::Arg::B(bo, off(ho))};+ d.grid[0] = (uint32_t) ((ncols + 255) / 256);+ d.grid[1] = (uint32_t) nrows;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+ }++ // kind: 0 BF16 values, 8 Q8_0, 13 Q5_K. n rows of k values, ld the row stride (bytes; values for BF16), out half rows.+ bool dequant_rows(int kind, const void* raw, size_t ld, int k, int n, uint16_t* out) {+ if (n <= 0 || k <= 0) return true;+ if (!ensure()) return false;+ const char* fn = kind == 13 ? "ds4_dq_q5k" : kind == 8 ? "ds4_dq_q8" : kind == 0 ? "ds4_dq_bf16" : nullptr;+ if (!fn) return false;+ if ((kind == 13 && k % 256 != 0) || (kind == 8 && k % 32 != 0)) return false;+ void* hw = strata::gpu::host_view((void*) raw);+ void* ho = strata::gpu::host_view(out);+ if (!hw || !ho) return false;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ const size_t wbytes = (size_t) (n - 1) * ld * (kind == 0 ? 2 : 1) + (kind == 0 ? (size_t) k * 2 : (kind == 13 ? (size_t) (k / 256) * 176 : (size_t) (k / 32) * 34));+ if (!std::getenv("DS4_NO_TOUCH")) { // (the matrix may be file-backed pages not read yet: the GPU does not page them in, it stalls)+ volatile unsigned char sink = 0;+ for (size_t o = 0; o < wbytes; o += 4096) sink = sink + ((const volatile unsigned char*) hw)[o];+ (void) sink;+ }+ mtl::Buffer bw = wrap(hw, wbytes);+ mtl::Buffer bo = wrap(ho, (size_t) n * (size_t) k * 2);+ if (!bw.handle || !bo.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, fn); } catch (...) { return false; }+ size_t items = kind == 13 ? (size_t) n * (k / 256) * 8 : kind == 8 ? (size_t) n * (k / 32) : (size_t) n * k;+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::V((unsigned long) ld), mtl::Arg::V((unsigned) k), mtl::Arg::V((unsigned) n),+ mtl::Arg::B(bo, off(ho))};+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ d.grid[0] = (uint32_t) ((items + 255) / 256);+ d.grid[1] = d.grid[2] = 1;+ g_cx->run({d});+ return true;+ }++ // Y[r][c] = alpha * (X16[r] . W16[c]) + beta * Y[r][c]: X16 rows x K (stride ldx halves), W16 N x K contiguous, Y rows x N float+ // (stride ldy). The cuBLAS call's operands in row-major terms; the MPP tile kernel wants contiguous operands and result.+ bool gemm_f16_nt(const uint16_t* W16, const uint16_t* X16, int ldx, float* Y, int ldy, int rows, int N, int K, float alpha,+ float beta) {+ if (rows <= 0 || N <= 0) return true;+ if (!ensure()) return false;+ if (g_lib_mm == -2) {+ g_lib_mm = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "dense_mm.metal"});+ if (g_lib_mm < 0) std::fprintf(stderr, "dsv41 metal: compiling dense_mm.metal failed\n");+ }+ if (g_lib_mm < 0) return false;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ void* hw = strata::gpu::host_view((void*) W16);+ void* hx = strata::gpu::host_view((void*) X16);+ void* hy = strata::gpu::host_view(Y);+ if (!hw || !hx || !hy) return false;+ mtl::Buffer bw = wrap(hw, (size_t) N * K * 2);+ mtl::Buffer bx = wrap(hx, ((size_t) (rows - 1) * (size_t) ldx + (size_t) K) * 2);+ mtl::Buffer by = wrap(hy, ((size_t) (rows - 1) * (size_t) ldy + (size_t) N) * 4);+ if (!bw.handle || !bx.handle || !by.handle) return false;+ // activations contiguous+ mtl::Buffer ax = bx;+ size_t ax_off = off(hx);+ if (ldx != K) {+ ax = scratch(g_gm_xs, (size_t) rows * K * 2);+ ax_off = 0;+ mtl::Pipeline pp;+ try { pp = g_cx->pipeline(g_lib, "ds4_pack_h16"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pp;+ d.args = {mtl::Arg::B(bx, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::V((unsigned) K), mtl::Arg::B(ax)};+ d.grid[0] = (uint32_t) ((K + 255) / 256);+ d.grid[1] = (uint32_t) rows;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ }+ const bool direct = ldy == N && alpha == 1.0f && beta == 0.0f;+ mtl::Buffer ty = by;+ size_t ty_off = off(hy);+ if (!direct) {+ ty = scratch(g_gm_yt, (size_t) rows * N * 4);+ ty_off = 0;+ }+ const int tm = 64, tn = N <= 64 ? 32 : 64;+ const std::string fn = "gemm_f16_" + std::to_string(tm) + "_" + std::to_string(tn) + "_4";+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib_mm, fn); } catch (...) { return false; }+ struct GemmArgs { int M, N, K, pad; } ga{rows, N, K, 0};+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(ax, ax_off), mtl::Arg::B(bw, off(hw)), mtl::Arg::B(ty, ty_off), mtl::Arg::V(ga)};+ d.grid[0] = (uint32_t) ((N + tn - 1) / tn);+ d.grid[1] = (uint32_t) ((rows + tm - 1) / tm);+ d.grid[2] = 1;+ d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ if (!direct) {+ mtl::Pipeline pa;+ try { pa = g_cx->pipeline(g_lib, "ds4_axpby_rows"); } catch (...) { return false; }+ mtl::Dispatch e;+ e.p = &pa;+ e.args = {mtl::Arg::B(ty), mtl::Arg::B(by, off(hy)), mtl::Arg::V((unsigned long) ldy), mtl::Arg::V((unsigned) N),+ mtl::Arg::V(alpha), mtl::Arg::V(beta)};+ e.grid[0] = (uint32_t) ((N + 255) / 256);+ e.grid[1] = (uint32_t) rows;+ e.grid[2] = 1;+ e.tg[0] = 256; e.tg[1] = e.tg[2] = 1;+ g_cx->run({e});+ }+ return true;+ }+++ // ---- the prompt chunk's expert GEMMs (ds4_mm.metal): a set = n_exp consecutive expert slots, their rows sorted by expert ----+ namespace {+ int g_lib_moe = -2;+ mtl::Buffer g_mm_xs, g_mm_hs, g_mm_ys; // grow-only scratch: the sorted rows' half activations; the gate/up's swiglu rows (the down's input)+ bool g_mm_hs_ok = false;-size_t iq_bs(int type) { return type == 16 ? 66 : type == 17 ? 74 : type == 22 ? 82 : type == 18 ? 98 : 0; }++// bytes of a 256-value block per type: the tile kernels are instantiated for exactly these (DS4_MM_INST)++size_t iq_bs(int type) { return type == 16 ? 66 : type == 17 ? 74 : type == 22 ? 82 : type == 18 ? 98 : type == 29 ? 56 : 0; }+ struct DsMmArgsH {+ unsigned long gate_off, up_off, down_off, stride;+ unsigned gu_row, d_row, n_embd, n_ff;+ float limit;+ unsigned pad0;+ };+ bool mm_prepare() {+ if (!ensure()) return false;+ if (g_lib_moe == -2) {+ g_lib_moe = g_cx->compile({"quant.metal", "units.metal", "mma.metal", "moe_mm.metal", "../src/engine/msl/ds4_mm.metal"});+ if (g_lib_moe < 0) std::fprintf(stderr, "dsv41 metal: compiling ds4_mm.metal failed\n");+ }+ return g_lib_moe >= 0;+ }+ } // namespace++ static bool mm_set(bool gate_up, const uint8_t* wbase, size_t stride, const size_t* tr, const int* hb, int n_exp, int total_rows,+ int n_embd, int n_ff, float limit, const float* xf, int ldx, const int* idx, const uint16_t* hin, uint16_t* hout,+ int ldh, float* yout, int ldy, int tg, int td) {+ const int type = gate_up ? tg : td;+ if (n_exp <= 0 || total_rows <= 0) return true;+ if (iq_bs(tg) == 0 || iq_bs(td) == 0 || n_embd % 256 != 0 || n_ff % 256 != 0) return false;+ if (ldh != n_ff || (!gate_up && ldy != n_embd)) return false;+ if (gate_up ? (n_ff % 128 != 0) : (n_embd % 128 != 0)) return false;+ if (!mm_prepare()) return false;- constexpr int TM = 16;++ constexpr int TM = 32; // (the tile kernels' TM: ds4_mm.metal DS4_MM_INST)++ constexpr int SG = 4, TN = 32; // the tile kernels' simdgroups and columns each:++ constexpr int COLS = SG * TN; // ds4_mm.metal DS4_MM_INST must match these+ // the tile list: (slot, first row, valid rows) a TM-row piece of one expert+ std::vector<int> tiles;+ for (int e = 0; e < n_exp; ++e) {+ const int lo = hb[e] - hb[0], hi = hb[e + 1] - hb[0];+ for (int r = lo; r < hi; r += TM) { tiles.push_back(e); tiles.push_back(r); tiles.push_back(std::min(TM, hi - r)); tiles.push_back(0); }+ }+ const int ntiles = (int) (tiles.size() / 4);+ if (ntiles == 0) return true;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ const size_t len = ((end - base + page - 1) / page) * page;+ if (std::getenv("DS4_MM_DBG")) { size_t o2 = 0; std::fprintf(stderr, "mm wrap %p len %zu own=%d rem=%zu\n", (void*) base, len, ds4_gpu_buffer_for_cpu((void*) base, len, &o2) != nullptr, ds4_gpu_alloc_remaining((void*) base)); }+ return g_cx->wrap((void*) base, len);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ void* hw = strata::gpu::host_view((void*) wbase);+ if (!hw) return false;+ mtl::Buffer bw = wrap(hw, (size_t) n_exp * stride);+ mtl::Buffer bt = g_cx->alloc(tiles.size() * 4);+ std::memcpy(bt.ptr, tiles.data(), tiles.size() * 4);+ DsMmArgsH a{tr[0], tr[1], tr[2], (unsigned long) stride, (unsigned) ((size_t) n_embd / 256 * iq_bs(tg)), (unsigned) ((size_t) n_ff / 256 * iq_bs(td)),+ (unsigned) n_embd, (unsigned) n_ff, limit, std::getenv("DS4_MM_ABL") ? (unsigned) std::atoi(std::getenv("DS4_MM_ABL")) : 0u};+ if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "mm %s embd %d ff %d tg %d td %d lim %g stride %zu tr %zu %zu %zu gu_row %u d_row %u\n", gate_up ? "gu" : "dn", n_embd, n_ff, tg, td, limit, stride, tr[0], tr[1], tr[2], a.gu_row, a.d_row);+ mtl::Pipeline pl;- try { pl = g_cx->pipeline(g_lib_moe, std::string(gate_up ? "ds4_mm_gu_" : "ds4_mm_dn_") + std::to_string(type)); } catch (...) { g_cx->free(bt); return false; }++ // DS4_MM_CTAB: QUANT_CTAB (function constant 23) - the i-quant grids and the sign table read from constant++ // memory instead of a threadgroup copy. Same values; the decode's two lookups per 8 weights go through the++ // constant cache instead of threadgroup memory.++ static const bool mm_ctab = std::getenv("DS4_MM_CTAB") != nullptr;++ try {++ const std::string fn = std::string(gate_up ? "ds4_mm_gu_" : "ds4_mm_dn_") + std::to_string(type);++ if (mm_ctab) {++ mtl::Constants cs;++ cs.b(23, true);++ pl = g_cx->pipeline_uncached(g_lib_moe, fn, cs);++ } else {++ pl = g_cx->pipeline(g_lib_moe, fn);++ }++ } catch (...) { g_cx->free(bt); return false; }+ if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "pipe %s max_threads %d tgmem %d width %d\n", pl.name.c_str(), pl.max_threads, pl.static_tg_mem, pl.simd_width);+ mtl::Dispatch d;+ d.p = &pl;- d.tg[0] = 128; d.tg[1] = d.tg[2] = 1;++ d.tg[0] = SG * 32; d.tg[1] = d.tg[2] = 1;+ d.grid[2] = 1;+ d.grid[1] = (uint32_t) ntiles;+ if (gate_up) {+ // the sorted rows' activations, half+ const size_t xs_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 2;+ if (!g_mm_xs.handle || g_mm_xs.bytes < xs_bytes) {+ if (g_mm_xs.handle) g_cx->free(g_mm_xs);+ g_mm_xs = g_cx->alloc(xs_bytes + (xs_bytes >> 2));+ }+ void* hx = strata::gpu::host_view((void*) xf);+ void* hi = strata::gpu::host_view((void*) idx);+ void* ho = strata::gpu::host_view(hout);+ if (!hx || !hi || !ho) { g_cx->free(bt); return false; }+ // (the float rows: the whole token slab the engine's c.x holds; its extent is the allocation's)+ const size_t xrem = ds4_gpu_alloc_remaining(hx);+ mtl::Buffer bxf = wrap(hx, xrem > 0 ? xrem : (size_t) ldx * 4);+ mtl::Buffer bix = wrap(hi, (size_t) total_rows * 4);+ mtl::Pipeline pg;+ try { pg = g_cx->pipeline(g_lib_moe, "ds4_mm_gather"); } catch (...) { g_cx->free(bt); return false; }+ mtl::Dispatch dg;+ dg.p = &pg;+ dg.args = {mtl::Arg::B(bxf, off(hx)), mtl::Arg::V((unsigned long) ldx), mtl::Arg::B(bix, off(hi)), mtl::Arg::V((unsigned) n_embd), mtl::Arg::B(g_mm_xs)};+ dg.grid[0] = (uint32_t) ((n_embd + 255) / 256);+ dg.grid[1] = (uint32_t) total_rows;+ dg.grid[2] = 1;+ dg.tg[0] = 256; dg.tg[1] = dg.tg[2] = 1;+ g_cx->run({dg});+ const size_t hs_bytes = ((size_t) total_rows + TM) * (size_t) n_ff * 2;+ if (!g_mm_hs.handle || g_mm_hs.bytes < hs_bytes) {+ if (g_mm_hs.handle) g_cx->free(g_mm_hs);+ g_mm_hs = g_cx->alloc(std::max<size_t>(hs_bytes + (hs_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));+ }+ g_mm_hs_ok = true;+ { // (the down's result rows are made here too)+ const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;+ if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {+ if (g_mm_ys.handle) g_cx->free(g_mm_ys);+ g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));+ }+ }+ (void) ho;+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_xs), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};- d.grid[0] = (uint32_t) (n_ff / 128);++ d.grid[0] = (uint32_t) (n_ff / COLS);+ } else {+ void* hh = strata::gpu::host_view((void*) hin);+ void* hy = strata::gpu::host_view(yout);+ void* hy_real = nullptr;+ if (!hh || !hy) { g_cx->free(bt); return false; }+ const size_t hrem = ds4_gpu_alloc_remaining(hh);+ if (!g_mm_hs_ok) { g_cx->free(bt); return false; }+ g_mm_hs_ok = false;+ (void) hrem;+ mtl::Buffer& bh = g_mm_hs;+ const size_t ys_bytes = ((size_t) total_rows + TM) * (size_t) n_embd * 4;+ if (!g_mm_ys.handle || g_mm_ys.bytes < ys_bytes) {+ if (g_mm_ys.handle) g_cx->free(g_mm_ys);+ g_mm_ys = g_cx->alloc(std::max<size_t>(ys_bytes + (ys_bytes >> 2), std::getenv("DS4_MM_BIG") ? (size_t) 1 << 26 : 0));+ }+ mtl::Buffer by = g_mm_ys;+ hy_real = hy;+ hy = by.ptr;+ if (std::getenv("DS4_MM_YALT")) { hy = hh; by = wrap(hh, (size_t) total_rows * (size_t) n_ff * 2); }+ if (std::getenv("DS4_MM_EXP1")) { by = g_cx->alloc((size_t) total_rows * (size_t) n_embd * 4 + 4096); hy = by.ptr; }+ d.args = {mtl::Arg::B(bw, off(hw)), mtl::Arg::B(g_mm_ys), mtl::Arg::B(g_mm_hs), mtl::Arg::B(bt), mtl::Arg::V(a), mtl::Arg::V((unsigned) ntiles)};- d.grid[0] = (uint32_t) (n_embd / 128);++ d.grid[0] = (uint32_t) (n_embd / COLS);+ }+ if (!gate_up && std::getenv("DS4_MM_ASGU")) { // test: the gate/up kernel on the down's buffers+ static mtl::Pipeline pgu;+ pgu = g_cx->pipeline(g_lib_moe, "ds4_mm_gu_" + std::to_string(tg));+ d.p = &pgu;+ d.args[1] = mtl::Arg::B(g_mm_xs);+ d.args[2] = mtl::Arg::B(g_mm_ys);- if (std::atoi(std::getenv("DS4_MM_ASGU")) == 1) d.grid[0] = (uint32_t) (n_ff / 128);++ if (std::atoi(std::getenv("DS4_MM_ASGU")) == 1) d.grid[0] = (uint32_t) (n_ff / COLS);+ }+ if (!gate_up && std::getenv("DS4_MM_DROP")) {+ const char* dk = std::getenv("DS4_MM_DROP");+ static mtl::Buffer dummy;+ if (!dummy.handle) dummy = g_cx->alloc(1 << 26);+ static mtl::Buffer dummy2;+ if (!dummy2.handle) dummy2 = g_cx->alloc(1 << 26);+ for (int k = 0; k < 4; ++k) if (std::strchr(dk, '0' + k)) d.args[k] = mtl::Arg::B(std::getenv("DS4_MM_D2") && k == std::atoi(std::getenv("DS4_MM_D2")) ? dummy2 : dummy);+ }+ if (!gate_up && std::getenv("DS4_MM_ONE")) { d.grid[1] = 1; if (std::getenv("DS4_MM_DBG")) std::fprintf(stderr, "dn tile0 %d %d %d ntiles %d x %u rows %d\n", tiles[0], tiles[1], tiles[2], ntiles, d.grid[0], total_rows); }+ if (std::getenv("DS4_MM_DBG")) for (size_t q = 0; q < d.args.size(); ++q) if (d.args[q].buf) std::fprintf(stderr, "arg %s %zu bytes %zu bias %zu off %zu ptr %p\n", gate_up ? "gu" : "dn", q, d.args[q].buf->bytes, (size_t) d.args[q].buf->bias, (size_t) d.args[q].off, d.args[q].buf->ptr);+ if (std::getenv(gate_up ? "DS4_MM_NOGU" : "DS4_MM_NODN")) { g_cx->free(bt); return true; }+ g_cx->run({d});+ if (!std::getenv("DS4_MM_LEAK")) g_cx->free(bt);+ if (!gate_up) { // the result into the engine's rows+ void* hyr = strata::gpu::host_view(yout);+ const size_t n4 = (size_t) total_rows * (size_t) n_embd;+ const size_t base = (size_t) ((uintptr_t) hyr) & ~(size_t) (page - 1);+ mtl::Buffer bd = g_cx->wrap((void*) base, ((((size_t) (uintptr_t) hyr + n4 * 4 - base) + page - 1) / page) * page);+ mtl::Pipeline pc;+ try { pc = g_cx->pipeline(g_lib, "ds4_copy4"); } catch (...) { return false; }+ mtl::Dispatch c;+ c.p = &pc;+ c.args = {mtl::Arg::B(g_mm_ys), mtl::Arg::B(bd, (size_t) ((uintptr_t) hyr - base)), mtl::Arg::V((unsigned long) n4)};+ c.tg[0] = 256; c.tg[1] = c.tg[2] = 1;+ c.grid[0] = (uint32_t) ((n4 + 255) / 256);+ c.grid[1] = c.grid[2] = 1;+ g_cx->run({c});+ }+ return true;+ }++ bool iq_set_gu(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,+ float limit, const float* xf, int ldx, const int* idx, uint16_t* h, int ldh, int tg, int td) {+ return mm_set(true, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, xf, ldx, idx, nullptr, h, ldh, nullptr, 0, tg, td);+ }+ bool iq_set_dn(const uint8_t* wbase, size_t stride, const size_t* tr, const int* hbounds, int n_exp, int total_rows, int n_embd, int n_ff,+ float limit, const uint16_t* h16, int ldh, float* y, int ldy, int tg, int td) {+ return mm_set(false, wbase, stride, tr, hbounds, n_exp, total_rows, n_embd, n_ff, limit, nullptr, 0, nullptr, h16, nullptr, ldh, y, ldy, tg, td);+ }++ // ---- copies, fills and event marks on the context's queue (see ds4_copy4 in ds4_init.metal) -------------------------------+ // strata::gpu routes cudaMemcpyAsync / cudaMemsetAsync here when both ends are its allocations; ds4_ctx_mark ends the batched+ // command buffer without waiting and returns a token the matching ds4_ctx_wait blocks on (cudaEventRecord / EventSynchronize).++// DS4_COPY_LOG: how much host time the copies take (they wrap host memory into Metal buffers per call)++namespace { struct CopyLog { unsigned long long n = 0, bytes = 0, us = 0;++ ~CopyLog() { if (n) std::fprintf(stderr, "copy log: %llu copies, %.1f MB, %.1f ms host time (%.1f us a copy)\n",++ n, bytes / 1e6, us / 1e3, n ? (double) us / (double) n : 0.0); } };++ CopyLog g_copy_us; }++static inline unsigned long long copy_now_us() {++ return (unsigned long long) std::chrono::duration_cast<std::chrono::microseconds>(++ std::chrono::steady_clock::now().time_since_epoch()).count();++}+ extern "C" bool ds4_ctx_copy(void* dst, const void* src, size_t n) {+ if (n == 0) return true;++ const bool copy_log = std::getenv("DS4_COPY_LOG") != nullptr;++ const unsigned long long copy_t0 = copy_log ? copy_now_us() : 0;++ struct CopyDone { bool on; unsigned long long t0; size_t n; ~CopyDone() { if (on) { g_copy_us.n++; g_copy_us.bytes += n; g_copy_us.us += copy_now_us() - t0; } } } copy_done{copy_log, copy_t0, n};+ if (!ensure()) return false;+ void* hd = strata::gpu::host_view(dst);+ void* hs = strata::gpu::host_view((void*) src);+ if (!hd || !hs) return false;+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ const size_t base = (size_t) ((uintptr_t) host) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) host) + bytes;+ return g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bs = wrap(hs, n);+ mtl::Buffer bd = wrap(hd, n);+ if (!bs.handle || !bd.handle) return false;+ const bool w4 = (((uintptr_t) hd | (uintptr_t) hs | n) & 3u) == 0;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, w4 ? "ds4_copy4" : "ds4_copy1"); } catch (...) { return false; }+ const size_t items = w4 ? n / 4 : n;+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bd, off(hd)), mtl::Arg::V((unsigned long) items)};+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ d.grid[0] = (uint32_t) ((items + 255) / 256);+ d.grid[1] = d.grid[2] = 1;+ g_cx->run({d});+ return true;+ }+ extern "C" bool ds4_ctx_fill(void* dst, int value, size_t n) {+ if (n == 0) return true;+ if (!ensure()) return false;+ void* hd = strata::gpu::host_view(dst);+ if (!hd) return false;+ const size_t page = 16384;+ const size_t base = (size_t) ((uintptr_t) hd) & ~(size_t) (page - 1);+ const size_t end = (size_t) ((uintptr_t) hd) + n;+ mtl::Buffer bd = g_cx->wrap((void*) base, ((end - base + page - 1) / page) * page);+ if (!bd.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_fill1"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bd, (size_t) ((uintptr_t) hd - base)), mtl::Arg::V((unsigned long) n), mtl::Arg::V((unsigned) (value & 0xff))};+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ d.grid[0] = (uint32_t) ((n + 255) / 256);+ d.grid[1] = d.grid[2] = 1;+ g_cx->run({d});+ return true;+ }+ extern "C" void* ds4_ctx_mark() {+ if (!g_cx) return nullptr;+ return g_cx->mark();+ }+ extern "C" void ds4_ctx_wait(void* token) {+ static const bool no_wait_split = std::getenv("DS4_NO_WAIT_SPLIT") != nullptr;+ if (g_cx && !no_wait_split) g_cx->split_runs(); // (what is queued after the mark starts on the GPU while the host waits)+ if (g_cx && token) g_cx->wait_mark(token);+ }+ // commit the batched dispatches without waiting: the GPU starts on them while the host goes on (the CPU lane)+ extern "C" void ds4_metal_kick() {+ if (g_cx) g_cx->split_runs();+ }++ extern "C" void ds4_metal_flush() {+ // The engine's copies (the embedding rows, the streamed experts, the readbacks) are encoded on strata::gpu's default+ // stream, its dispatched kernels on mtl_rt's deferred command buffer. Two queues, so committing one alone leaves the+ // other's work unordered: a kernel could read a feed whose blit had not run yet (emb_rows is a zeroed buffer, so+ // embed wrote exactly 0.0 into h). The stream's queue goes first - it only carries the feeds at a token boundary,+ // the readbacks are encoded after this returns.+ ds4_gpu_sync_all_streams();+ if (g_cx) g_cx->flush_runs();+ }+++ // the one Context every dsv41 kernel dispatches through: two of them means two Metal queues and no ordering between their work+ mtl::Context& shared_cx() {+ ensure();+ return *g_cx;+ }+ mtl::Context& finish_cx() {+ ensure();+ return *g_cx;+ }++ int shared_lib() {+ ensure();+ return g_lib;+ }++ } // namespace strata::dsv41::metal_hostdiff --cc metal/src/engine/msl/ds4_init.metalindex 000000000,70761a4ae..ed98143a6mode 000000,100644..100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@@ -1,0 -1,2408 +1,2770 @@@+ // metal/src/engine/msl/ds4_init.metal - the first kernels the engine asks for, ported from ds4_kernels.cu.+ //+ // ds4_kernels.cu:1477 __global__ void init_pre_kernel(float* pre, int T) {+ // const int i = blockIdx.x * blockDim.x + threadIdx.x;+ // if (i < T * 8) pre[i] = (i % 8) == 0 ? 1.0f : 0.0f;+ // }+ #include <metal_stdlib>++ using namespace metal;++ // the grid is threadgroups of 128; the flat id is what CUDA's blockIdx*blockDim+threadIdx was+ kernel void ds4_init_pre(device float* pre [[buffer(0)]], constant uint& T [[buffer(1)]],+ uint gid [[thread_position_in_grid]]) {+ if (gid < T * 8u) pre[gid] = (gid % 8u) == 0u ? 1.0f : 0.0f;+ }++ // ds4_kernels.cu:130 embed_kernel: v = bf16(rows[t*dim + e]); h[(t*hc + s)*dim + e] = v for each of hc copies+ kernel void ds4_embed(device const ushort* rows [[buffer(0)]], device float* h [[buffer(1)]],+ constant uint& dim [[buffer(2)]], constant uint& hc [[buffer(3)]],+ uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + uint(lane);+ if (e >= dim) return;+ const uint t = tgp.y;+ const uint bits = uint(rows[(size_t) t * dim + e]) << 16; // BF16 -> F32 is the top half+ const float v = as_type<float>(bits);+ for (uint s = 0; s < hc; ++s) h[((size_t) t * hc + s) * dim + e] = v;+ }++ // ds4_model.cu:2737 cublasSgemm(OP_T, OP_N, n_mix, T, hcd, ...) - the mHC mixes: mixes[t][m] = h[t] . fn[m]+ // one thread an output element; both operands F32, alpha = 1, beta = 0 (every call site seen)+ kernel void ds4_sgemm_tn(device const float* A [[buffer(0)]], device const float* B [[buffer(1)]],+ device float* C [[buffer(2)]], constant uint& m [[buffer(3)]], constant uint& nt [[buffer(4)]],+ constant uint& k [[buffer(5)]], constant uint& lda [[buffer(6)]], constant uint& ldb [[buffer(7)]],+ constant uint& ldc [[buffer(8)]], uint gid [[thread_position_in_grid]]) {+ const uint total = m * nt;+ if (gid >= total) return;+ const uint t = gid / m, mm = gid % m;+ const device float* a = A + (size_t) mm * lda; // row mm of A+ const device float* b = B + (size_t) t * ldb; // row t of B+ float acc = 0.0f;+ for (uint i = 0; i < k; ++i) acc += a[i] * b[i];+ C[(size_t) t * ldc + mm] = acc;+ }++ // ds4_kernels.cu:1894 finish_kernel<false> - RMS norm (f.w), mrope (f.rd), rounding (f.quant), then store to x or dst.+ // The mrope and rounding branches are not written yet; the host wrapper will decline until they are.+ struct Ds4Finish {+ device float* w;+ float eps;+ int rd, pos, inverse, vpp, pos_step;+ device float* freqs;+ int quant;+ device ushort* dst;+ long dst_stride;+ device uchar* pdst;+ long pdst_stride;+ };++ static inline float ds4_warp_sum(float v) {+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ return v;+ }+ static inline float ds4_block_sum(float v, threadgroup float* red, ushort lane, ushort w) {+ v = ds4_warp_sum(v);+ if (lane == 0) red[w] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (w == 0) {+ float t = lane < 8 ? red[lane] : 0.0f;+ t = ds4_warp_sum(t);+ if (lane == 0) red[0] = t;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return red[0];+ }++ // ds4_kernels.cu:65 pow2_ceil / :88 fp8_rt / :90 e2m1_rt - the KV quantiser's scale and roundings, shared by finish_pack+ // (the packed rows) and ds4_finish's rounding branch+ static inline float ds4_pow2_ceil(float x) {+ const uint b = as_type<uint>(x);+ const int e = int((b >> 23) & 0xffu) - 127 + ((b & 0x7fffffu) != 0u ? 1 : 0);+ return as_type<float>(uint(e + 127) << 23);+ }++ static inline float ds4_e4m3_round(float v) { // E4M3, nearest even (the input is clamped)+ const float a = fabs(v);+ if (!(a == a)) return v;+ float r;+ if (a >= 448.0f) r = 448.0f;+ else {+ const int e = int((as_type<uint>(a) >> 23) & 0xffu) - 127;+ const float step = e < -6 ? 0.001953125f : as_type<float>(uint(e - 3 + 127) << 23);+ r = a == 0.0f ? 0.0f : rint(a / step) * step;+ r = fmin(r, 448.0f);+ }+ return copysign(r, v);+ }++ static inline float ds4_e2m1_rt(float v) { // E2M1 magnitudes 0 .5 1 1.5 2 3 4 6, nearest, ties to even+ const float a = fabs(v);+ float r;+ if (a <= 0.25f) r = 0.0f;+ else if (a < 0.75f) r = 0.5f;+ else if (a <= 1.25f) r = 1.0f;+ else if (a < 1.75f) r = 1.5f;+ else if (a <= 2.5f) r = 2.0f;+ else if (a < 3.5f) r = 3.0f;+ else if (a <= 5.0f) r = 4.0f;+ else r = 6.0f;+ return copysign(r, v);+ }++ kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],+ constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]], uint b [[threadgroup_position_in_grid]]) {+ threadgroup float red[8];+ device float* v = x + (long) b * stride;+ const uint nj = (len + 255u) / 256u;+ float r[32];+ float ss = 0.0f;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ r[j] = e < len ? v[e] : 0.0f;+ ss += r[j] * r[j];+ }+ if (f.w != nullptr) {+ const float tot = ds4_block_sum(ss, red, lane, w);+ const float inv = rsqrt(tot / float(len) + f.eps);+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ if (e < len) r[j] = r[j] * inv * f.w[e];+ }+ }+ if (f.rd > 0) {+ // mrope: the last rd values move in (re, im) pairs a lane apart (ds4_kernels.cu:161): the partner comes from a+ // lane xor 1, the angle from the token's position and the frequency of the pair+ const uint r0 = len - uint(f.rd);+ const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ const float other = simd_shuffle_xor(r[j], 1);+ if (e >= r0 && e < len) {+ const uint i = (e - r0) >> 1;+ const float ang = float(pos) * f.freqs[i];+ const float c = cos(ang);+ const float sn = f.inverse ? -sin(ang) : sin(ang);+ r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;+ }+ }+ }+ if (f.quant != 0) { // the KV rounding without the pack: the group's scale, then the value rounded to the format+ const uint g = f.quant == 2 ? 16u : 32u;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ float a = e < len ? fabs(r[j]) : 0.0f;+ for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));+ if (e >= len) continue;+ if (f.quant == 1) {+ const float s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));+ r[j] = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f)) * s;+ } else if (f.quant == 2) {+ const float s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);+ r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;+ } else {+ const float s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));+ r[j] = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f)) * s;+ }+ }+ }+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ if (e >= len) continue;+ if (f.dst != nullptr) {+ const uint bits = as_type<uint>(r[j]);+ f.dst[(long) b * f.dst_stride + (long) e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even, the vector's own row+ } else {+ v[e] = r[j];+ }+ }+ }++ // ds4_kernels.cu:1677 hc_post_inplace_kernel: h[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*h_old[t][j][e]+ kernel void ds4_hc_post(device const float* y [[buffer(0)]], device float* h [[buffer(1)]],+ device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],+ constant uint& dim [[buffer(4)]], constant uint& hc [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + uint(lane);+ if (e >= dim) return;+ const uint t = tgp.y;+ device float* r = h + (size_t) t * hc * dim;+ float rv[8];+ for (uint j = 0; j < 8u; ++j) if (j < hc) rv[j] = r[(size_t) j * dim + e];+ const float yv = y[(size_t) t * dim + e];+ for (uint k = 0; k < 8u; ++k) {+ if (k >= hc) break;+ float a = post[(size_t) t * 8u + k] * yv;+ for (uint j = 0; j < 8u; ++j) if (j < hc) a += comb[(size_t) t * 64u + j * hc + k] * rv[j];+ r[(size_t) k * dim + e] = a;+ }+ }++ // ds4_kernels.cu:1692 hc_mix_scale_kernel: mixes[t][i] *= rsqrt(sum(h[t])^2 / hcdim + eps), one threadgroup a token+ kernel void ds4_hc_mix_scale(device const float* h [[buffer(0)]], device float* mixes [[buffer(1)]],+ constant uint& hcdim [[buffer(2)]], constant uint& n_mix [[buffer(3)]],+ constant float& eps [[buffer(4)]], uint b [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[8];+ const device float* x = h + (size_t) b * hcdim;+ float ss = 0.0f;+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {+ const float4 hv = *(const device float4*) (x + k);+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;+ }+ ss = ds4_block_sum(ss, red, lane, w);+ const float inv = rsqrt(ss / float(hcdim) + eps);+ if (tid < n_mix) mixes[(size_t) b * 32u + tid] *= inv;+ }++ // ds4_kernels.cu:1017 hc_mix_kernel: mixes[t][m] = (h[t] . fn[m]) * rsqrt(mean(h[t]^2) + eps), one threadgroup a (m, t)+ kernel void ds4_hc_mix(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],+ device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],+ constant float& eps [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[8];+ const uint m = tgp.x, t = tgp.y;+ const device float* x = h + (size_t) t * hcdim;+ const device float* f = fn + (size_t) m * hcdim;+ float acc = 0.0f, ss = 0.0f;+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {+ const float4 hv = *(const device float4*) (x + k);+ const float4 fv = *(const device float4*) (f + k);+ acc += hv.x * fv.x + hv.y * fv.y + hv.z * fv.z + hv.w * fv.w;+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;+ }+ acc = ds4_block_sum(acc, red, lane, w);+ ss = ds4_block_sum(ss, red, lane, w);+ if (tid == 0) mixes[(size_t) t * 32u + m] = acc * rsqrt(ss / float(hcdim) + eps);+ }++ // hc_mix for a prompt chunk: one threadgroup a token for four mixes a pass (the token's row is read once per four mixes instead of once per+ // mix). Every mix's partial sums keep the single-mix kernel's order, so the result is bit-identical.+ kernel void ds4_hc_mix_all(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],+ device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],+ constant float& eps [[buffer(4)]], constant uint& n_mix [[buffer(5)]],+ uint t [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[8];+ const device float* x = h + (size_t) t * hcdim;+ float ss = 0.0f;+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {+ const float4 hv = *(const device float4*) (x + k);+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;+ }+ ss = ds4_block_sum(ss, red, lane, w);+ const float inv = rsqrt(ss / float(hcdim) + eps);+ for (uint m0 = 0; m0 < n_mix; m0 += 4u) {+ const uint nm = min(4u, n_mix - m0);+ float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f;+ const device float* f0 = fn + (size_t) m0 * hcdim;+ const device float* f1 = f0 + (nm > 1u ? hcdim : 0u);+ const device float* f2 = f0 + (nm > 2u ? 2u * hcdim : 0u);+ const device float* f3 = f0 + (nm > 3u ? 3u * hcdim : 0u);+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {+ const float4 hv = *(const device float4*) (x + k);+ const float4 v0 = *(const device float4*) (f0 + k);+ const float4 v1 = *(const device float4*) (f1 + k);+ const float4 v2 = *(const device float4*) (f2 + k);+ const float4 v3 = *(const device float4*) (f3 + k);+ a0 += hv.x * v0.x + hv.y * v0.y + hv.z * v0.z + hv.w * v0.w;+ a1 += hv.x * v1.x + hv.y * v1.y + hv.z * v1.z + hv.w * v1.w;+ a2 += hv.x * v2.x + hv.y * v2.y + hv.z * v2.z + hv.w * v2.w;+ a3 += hv.x * v3.x + hv.y * v3.y + hv.z * v3.z + hv.w * v3.w;+ }+ a0 = ds4_block_sum(a0, red, lane, w);+ a1 = ds4_block_sum(a1, red, lane, w);+ a2 = ds4_block_sum(a2, red, lane, w);+ a3 = ds4_block_sum(a3, red, lane, w);+ if (tid == 0) {+ mixes[(size_t) t * 32u + m0] = a0 * inv;+ if (nm > 1u) mixes[(size_t) t * 32u + m0 + 1u] = a1 * inv;+ if (nm > 2u) mixes[(size_t) t * 32u + m0 + 2u] = a2 * inv;+ if (nm > 3u) mixes[(size_t) t * 32u + m0 + 3u] = a3 * inv;+ }+ }+ }+++// hc mix + split in one dispatch. The mix is ds4_hc_mix_all's (four mixes a pass, every one keeping the single-mix++// kernel's summation order, so the values are bit-identical), then the same threadgroup runs the sinkhorn split of the++// 24 mixes into pre / post / comb. Two dispatches become one, and nothing else changes.++kernel void ds4_hc_mix_split(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],++ device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],++ constant float& eps [[buffer(4)]], constant uint& n_mix [[buffer(5)]],++ device const float* scale [[buffer(6)]], device const float* base [[buffer(7)]],++ device float* pre [[buffer(8)]], device float* post [[buffer(9)]],++ device float* comb [[buffer(10)]], constant uint& hc [[buffer(11)]],++ constant uint& iters [[buffer(12)]], constant float& hc_eps [[buffer(13)]],++ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],++ ushort lane [[thread_index_in_simdgroup]], ushort wid [[simdgroup_index_in_threadgroup]]) {++ threadgroup float red[8];++ const device float* x = h + (size_t) t * hcdim;++ float ss = 0.0f;++ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {++ const float4 hv = *(const device float4*) (x + k);++ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;++ }++ ss = ds4_block_sum(ss, red, lane, wid);++ const float inv = rsqrt(ss / float(hcdim) + eps);++ for (uint m0 = 0; m0 < n_mix; m0 += 4u) {++ const uint nm = min(4u, n_mix - m0);++ float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f;++ const device float* f0 = fn + (size_t) m0 * hcdim;++ const device float* f1 = f0 + (nm > 1u ? hcdim : 0u);++ const device float* f2 = f0 + (nm > 2u ? 2u * hcdim : 0u);++ const device float* f3 = f0 + (nm > 3u ? 3u * hcdim : 0u);++ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {++ const float4 hv = *(const device float4*) (x + k);++ const float4 v0 = *(const device float4*) (f0 + k);++ const float4 v1 = *(const device float4*) (f1 + k);++ const float4 v2 = *(const device float4*) (f2 + k);++ const float4 v3 = *(const device float4*) (f3 + k);++ a0 += hv.x * v0.x + hv.y * v0.y + hv.z * v0.z + hv.w * v0.w;++ a1 += hv.x * v1.x + hv.y * v1.y + hv.z * v1.z + hv.w * v1.w;++ a2 += hv.x * v2.x + hv.y * v2.y + hv.z * v2.z + hv.w * v2.w;++ a3 += hv.x * v3.x + hv.y * v3.y + hv.z * v3.z + hv.w * v3.w;++ }++ a0 = ds4_block_sum(a0, red, lane, wid);++ a1 = ds4_block_sum(a1, red, lane, wid);++ a2 = ds4_block_sum(a2, red, lane, wid);++ a3 = ds4_block_sum(a3, red, lane, wid);++ if (tid == 0) {++ mixes[(size_t) t * 32u + m0] = a0 * inv;++ if (nm > 1u) mixes[(size_t) t * 32u + m0 + 1u] = a1 * inv;++ if (nm > 2u) mixes[(size_t) t * 32u + m0 + 2u] = a2 * inv;++ if (nm > 3u) mixes[(size_t) t * 32u + m0 + 3u] = a3 * inv;++ }++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // the split: ds4_hc_split4's math, in the first warp (every lane of it takes the same path)++ const device float* mx = mixes + (size_t) t * 32u;++ if (tid < 4u) {++ pre[(size_t) t * 8u + tid] = 1.0f / (1.0f + exp(-(mx[tid] * scale[0] + base[tid]))) + hc_eps;++ post[(size_t) t * 8u + tid] = 2.0f / (1.0f + exp(-(mx[4u + tid] * scale[1] + base[4u + tid])));++ }++ if (tid < 32u) {++ const bool on = lane < 16u;++ float c = on ? mx[8u + lane] * scale[2] + base[8u + lane] : -3.402823466e+38f;++ float m = fmax(c, simd_shuffle_xor(c, ushort(1)));++ m = fmax(m, simd_shuffle_xor(m, ushort(2)));++ c = on ? exp(c - m) : 0.0f;++ float rs = c + simd_shuffle_xor(c, ushort(1));++ rs += simd_shuffle_xor(rs, ushort(2));++ c = c / rs + hc_eps;++ c = on ? c : 0.0f;++ float cs = c + simd_shuffle_xor(c, ushort(4));++ cs += simd_shuffle_xor(cs, ushort(8));++ c = c / (cs + hc_eps);++ for (uint it = 1; it < iters; ++it) {++ float r = c + simd_shuffle_xor(c, ushort(1));++ r += simd_shuffle_xor(r, ushort(2));++ c = c / (r + hc_eps);++ float q = c + simd_shuffle_xor(c, ushort(4));++ q += simd_shuffle_xor(q, ushort(8));++ c = c / (q + hc_eps);++ }++ if (on) comb[(size_t) t * 64u + lane] = c;++ }++}+++ // ds4_kernels.cu:268 hc_split_kernel - the sinkhorn normalisation of the hyper-connection mix.+ // This is the SCALAR path of the CUDA kernel (lane 0 of the threadgroup does the token, a local 8x8 matrix), applied for every+ // hc including 4: it computes the same mathematics as the warp-lane version, only without the shuffles. Correctness first -+ // the lane-parallel hc == 4 version can be added behind a profile.+ kernel void ds4_hc_split(device const float* mixes [[buffer(0)]], device const float* scale [[buffer(1)]],+ device const float* base [[buffer(2)]], device float* pre [[buffer(3)]],+ device float* post [[buffer(4)]], device float* comb [[buffer(5)]],+ constant uint& hc [[buffer(6)]], constant uint& iters [[buffer(7)]],+ constant float& eps [[buffer(8)]], uint b [[threadgroup_position_in_grid]],+ uint lane [[thread_index_in_threadgroup]]) {+ if (lane != 0u) return;+ const device float* mx = mixes + (size_t) b * 32u;+ device float* pr = pre + (size_t) b * 8u;+ device float* po = post + (size_t) b * 8u;+ device float* cb = comb + (size_t) b * 64u;+ for (uint i = 0; i < 8u; ++i) {+ if (i >= hc) break;+ pr[i] = 1.0f / (1.0f + exp(-(mx[i] * scale[0] + base[i]))) + eps;+ po[i] = 2.0f / (1.0f + exp(-(mx[hc + i] * scale[1] + base[hc + i])));+ }+ float c[8][8];+ for (uint j = 0; j < 8u; ++j) {+ if (j >= hc) break;+ float m = -3.402823466e+38f;+ for (uint k = 0; k < 8u; ++k) {+ if (k >= hc) break;+ c[j][k] = mx[2u * hc + j * hc + k] * scale[2] + base[2u * hc + j * hc + k];+ m = fmax(m, c[j][k]);+ }+ float s = 0.0f;+ for (uint k = 0; k < 8u; ++k) {+ if (k >= hc) break;+ c[j][k] = exp(c[j][k] - m);+ s += c[j][k];+ }+ for (uint k = 0; k < 8u; ++k) {+ if (k >= hc) break;+ c[j][k] = c[j][k] / s + eps;+ }+ }+ for (uint k = 0; k < 8u; ++k) {+ if (k >= hc) break;+ float s = 0.0f;+ for (uint j = 0; j < 8u; ++j) if (j < hc) s += c[j][k];+ for (uint j = 0; j < 8u; ++j) if (j < hc) c[j][k] /= s + eps;+ }+ for (uint it = 1; it < iters; ++it) {+ for (uint j = 0; j < 8u; ++j) {+ if (j >= hc) break;+ float s = 0.0f;+ for (uint k = 0; k < 8u; ++k) if (k < hc) s += c[j][k];+ for (uint k = 0; k < 8u; ++k) if (k < hc) c[j][k] /= s + eps;+ }+ for (uint k = 0; k < 8u; ++k) {+ if (k >= hc) break;+ float s = 0.0f;+ for (uint j = 0; j < 8u; ++j) if (j < hc) s += c[j][k];+ for (uint j = 0; j < 8u; ++j) if (j < hc) c[j][k] /= s + eps;+ }+ }+ for (uint j = 0; j < 8u; ++j) {+ if (j >= hc) break;+ for (uint k = 0; k < 8u; ++k) { if (k >= hc) break; cb[j * hc + k] = c[j][k]; }+ }+ }++ // ds4_kernels.cu:1938 moe_sort - count each virtual slot's picks, prefix-sum the counts, then fill in token order.+ // kPicks = 8 (ds4_kernels.cuh:19): idx is [T][8], so q -> (t = q/8, i = q%8).+ kernel void ds4_bounds_scan(device int* bounds [[buffer(0)]], constant uint& n_vs [[buffer(1)]],+ uint tid [[thread_index_in_threadgroup]]) {+ if (tid != 0u) return;+ bounds[0] = 0;+ for (uint s = 0; s < n_vs; ++s) bounds[s + 1] += bounds[s];+ }++ // simd_ballot returns a simd_vote, which does not cast to an integer in this toolchain; the same counts come from the+ // prefix-sum and sum primitives, which give a lane its offset inside its simdgroup and the group its total.+ static inline int ds4_warp_prefix(bool hit) { return simd_prefix_exclusive_sum(hit ? 1 : 0); }+ static inline int ds4_warp_total(bool hit) { return simd_sum(hit ? 1 : 0); }++ kernel void ds4_moe_sort_count(device const int* idx [[buffer(0)]], constant uint& T [[buffer(1)]],+ constant uint& kk [[buffer(2)]], device const int* vs [[buffer(3)]],+ device int* bounds [[buffer(4)]], uint slot [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ threadgroup int warp_n[8];+ threadgroup int base_s;+ const uint wp = tid / 32u, lane = tid % 32u;+ if (tid == 0u) base_s = 0;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const uint n = T * kk;+ for (uint q = tid; q < n; q += 256u) {+ const uint t = q / kk, i = q - t * kk;+ const bool hit = vs[idx[t * 8u + i]] == int(slot);+ // one vote a warp: the running offset needs each warp's count, accumulated in slot order+ const int wtot = ds4_warp_total(hit);+ if ((tid % 32u) == 0u) warp_n[wp] = wtot;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ int total = 0;+ for (uint w = 0; w < 8u; ++w) total += warp_n[w];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid == 0u) base_s += total;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ if (tid == 0u) bounds[slot + 1] = base_s;+ }++ kernel void ds4_moe_sort_fill(device const int* idx [[buffer(0)]], constant uint& T [[buffer(1)]],+ constant uint& kk [[buffer(2)]], device const int* vs [[buffer(3)]],+ device int* bounds [[buffer(4)]], device int* row_tok [[buffer(5)]],+ device int* pos [[buffer(6)]], uint slot [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ threadgroup int warp_n[8];+ threadgroup int base_s;+ const uint wp = tid / 32u, lane = tid % 32u;+ if (tid == 0u) base_s = bounds[slot];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const uint n = T * kk;+ for (uint q = tid; q < n; q += 256u) {+ const uint t = q / kk, i = q - t * kk;+ const bool hit = vs[idx[t * 8u + i]] == int(slot);+ const int wpre = ds4_warp_prefix(hit), wtot = ds4_warp_total(hit);+ if ((tid % 32u) == 0u) warp_n[wp] = wtot;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ int before = 0, total = 0;+ for (uint w = 0; w < 8u; ++w) { if (w < wp) before += warp_n[w]; total += warp_n[w]; }+ if (hit) {+ const int r = base_s + before + wpre;+ row_tok[r] = int(t);+ pos[t * 8u + i] = r;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid == 0u) base_s += total;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ }++ // ds4_kernels.cu:1690 engram_dequant_tok_kernel - one hash-table row a threadgroup: IQ4_NL blocks to F32.+ // fmt 2 (this model's Engram): an 18-byte block is fp16 d then 16 packed nibbles; the low nibble of q[j] is value j and the+ // high nibble value j + 16, each an index into ggml-common.h's kvalues_iq4nl (ds4_kernels.cu:1280).+ constant int ds4_kIq4nl[16] = {-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};++ kernel void ds4_engram_dequant(device const uchar* src [[buffer(0)]], constant ulong& per [[buffer(1)]],+ constant uint& cols [[buffer(2)]], constant uint& hd [[buffer(3)]],+ device float* out [[buffer(4)]], constant int& fmt [[buffer(5)]],+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint t = r / cols, c = r - t * cols;+ const device uchar* rows = src + (size_t) t * per + (size_t) c * hd;+ if (fmt == 2) {+ for (uint e = tid; e < hd; e += 256u) {+ const device uchar* b = rows + (size_t) (e / 32u) * 18u;+ const uint q = b[2u + (e & 15u)];+ const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);+ const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);+ out[(size_t) r * hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);+ }+ return;+ }+ // fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: write NaN, so a wrong path shows as NaN in the output rather than+ // as a silent zero.+ for (uint e = tid; e < hd; e += 256u) out[(size_t) r * hd + e] = NAN;+ }++ // ds4_kernels.cu:341 hc_pre_norm_kernel - per token: a = sum_s mix[s]*h[token][s][e], one RMS over dim, then x = a*inv+ // (times w when given). One threadgroup a token, 1024 threads; a thread holds at most 8 values (dim <= 8*1024 = 8192).+ kernel void ds4_hc_pre_norm(device const float* h [[buffer(0)]], device const float* mix [[buffer(1)]],+ device const float* w [[buffer(2)]], constant uint& dim [[buffer(3)]],+ constant uint& hc [[buffer(4)]], constant float& eps [[buffer(5)]],+ device float* x [[buffer(6)]], constant uint& has_w [[buffer(7)]],+ uint b [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[32];+ const device float* hh = h + (size_t) b * hc * dim;+ const device float* mm = mix + (size_t) b * 8u;+ device float* xx = x + (size_t) b * dim;+ float r[8];+ float ss = 0.0f;+ uint n = 0;+ for (uint e = tid; e < dim; e += 1024u, ++n) {+ float a = 0.0f;+ for (uint s = 0; s < 8u; ++s) if (s < hc) a += mm[s] * hh[(size_t) s * dim + e];+ if (n < 8u) r[n] = a;+ ss += a * a;+ }+ // a 1024-thread reduction: 32 simdgroups, each into red[], then the first simdgroup sums those+ float v = ds4_warp_sum(ss);+ if (lane == 0u) red[wg] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (wg == 0u) {+ float t = lane < 32u ? red[lane] : 0.0f;+ t = ds4_warp_sum(t);+ if (lane == 0u) red[0] = t;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const float inv = rsqrt(red[0] / float(dim) + eps);+ n = 0;+ for (uint e = tid; e < dim; e += 1024u, ++n) {+ if (n >= 8u) break;+ xx[e] = has_w != 0u ? r[n] * inv * w[e] : r[n];+ }+ }++ // ds4_gguf.cu:186 q8_quant_kernel: a warp a 32-group - amax across the warp, sc = amax/127, q = rint(v/sc), d = sc.+ // grid = (ceil((k/32) / 8 warps), rows), threadgroup = 256 (8 warps of 32).+ kernel void ds4_q8_quant(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],+ constant uint& k [[buffer(2)]], device char* q [[buffer(3)]],+ device float* d [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const uint warp = tid / 32u, lane = tid % 32u;+ const uint b = tgp.x * 8u + warp; // the 32-group index+ const uint r = tgp.y;+ if ((size_t) b * 32u >= (size_t) k) return;+ const device float* xr = x + (size_t) r * ldx;+ const float v = xr[(size_t) b * 32u + lane];+ const float am = simd_max(fabs(v));+ const float sc = am / 127.0f;+ q[(size_t) r * k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);+ if (lane == 0u) d[(size_t) r * (k / 32u) + b] = sc;+ }++ // ds4_gguf.cu:371 mx_mv_kernel - the MXFP4 expert matrix-vector.+ // grid = (ceil(n / 8 warps), jobs), threadgroup = 256. A warp a row; a lane strides the 32-value blocks; NC columns of+ // activations share each decoded block (the T=8 window), then a warp sum and y[c][row] = alpha*v + bias[row].+ // Block layout (kMxBytes = 17 a 32 values): the E8M0 scale byte, then 16 bytes whose low nibble is value j and high nibble+ // value j+16, each an index into the 16-entry codebook {0,1,2,3,4,6,8,12} and its negatives.+ struct Ds4MxJob {+ device const void* w;+ ulong ld; // bytes a row+ device const float* x;+ ulong xcs;+ device float* y;+ ulong ycs;+ device const float* bias;+ float alpha;+ int k, n, ncol;+ device const char* xq;+ device const float* xd;+ };++ static inline float ds4_e8m0(uint e) { // 2^(e - 128), ggml's GGML_E8M0_TO_FP32_HALF+ return as_type<float>(e < 2u ? (0x00200000u << e) : ((e - 1u) << 23));+ }++ static inline void ds4_mx_block(device const uchar* p, threadgroup const float* lut, thread float* w) {+ const float d = ds4_e8m0(uint(p[0]));+ for (uint j = 0; j < 16u; ++j) {+ const uint q = uint(p[1u + j]);+ w[j] = lut[q & 15u] * d;+ w[j + 16u] = lut[q >> 4] * d;+ }+ }++ kernel void ds4_mx_mv(device const uchar* jobs [[buffer(0)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]]) {+ threadgroup float lut[16];+ if (tid < 16u) {+ const float v[8] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 6.0f, 8.0f, 12.0f};+ lut[tid] = tid < 8u ? v[tid] : -v[tid - 8u];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;+ thread float acc[8];+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;+ const uint nblk = uint(b.k) / 32u;+ for (uint kb = lane; kb < nblk; kb += 32u) {+ float wv[32];+ ds4_mx_block(w + (size_t) kb * 17u, lut, wv);+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ const device float* x = b.x + c * b.xcs + (size_t) kb * 32u;+ float a = acc[c];+ for (uint j = 0; j < 32u; ++j) a = fma(wv[j], x[j], a);+ acc[c] = a;+ }+ }+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+ }++ // ds4_gguf.cu:320 q8_mv_kernel - the Q8_0 matvec against quantised activations.+ // grid = (ceil(n / 8), jobs), threadgroup 256: a warp a row, a lane strides the 34-byte blocks (fp16 d + 32 int8), the dot is+ // integer (CUDA's __dp4a in four-value groups; here the same sum written out), then acc += (dw * xd[c]) * sum.+ // The staged-prefetch (ST) and vector variants of the CUDA kernel are prefetch strategies only - this reads the blocks directly.+ kernel void ds4_q8_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const device uchar* w = (const device uchar*) b.w + (size_t) row * b.ld;+ const uint nb = uint(b.k) / 32u;+ float acc[8];+ for (uint c = 0; c < 8u; ++c) acc[c] = 0.0f;+ for (uint kb = lane; kb < nb; kb += 32u) {+ const device uchar* p = w + (size_t) kb * 34u;+ const ushort dh = ushort(p[0]) | (ushort(p[1]) << 8);+ const float dw = float(as_type<half>(dh));+ int wv[32];+ for (uint j = 0; j < 32u; ++j) wv[j] = int((char) p[2u + j]);+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + (size_t) kb * 32u;+ int sum = 0;+ for (uint j = 0; j < 32u; ++j) sum += wv[j] * int(xq[j]);+ acc[c] = fma(dw * b.xd[(size_t) c * nb + kb], float(sum), acc[c]);+ }+ }+ for (uint c = 0; c < 8u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+ }++ // ds4_kernels.cu:1092 ring_to_ext_kernel - one ring row of the KV cache to BF16: each value an E4M3 code times its group's+ // E8M0 power of two (row[e] and row[d + e/32]); rows before the window start write zero.+ static inline float ds4_e4m3(uint b) { // E4M3 (bias 7, no infinities, max 448): sign, 4-bit exp, 3-bit mantissa+ const float s = (b & 0x80u) ? -1.0f : 1.0f;+ const uint ex = (b >> 3) & 0xfu, m = b & 7u;+ if (ex == 0u) return s * float(m) * (1.0f / 8.0f) * (1.0f / 64.0f);+ if (ex == 15u && m == 7u) return NAN;+ return s * exp2(float(ex) - 7.0f) * (1.0f + float(m) * 0.125f);+ }+ static inline float ds4_ue8m0(uint b) { return as_type<float>(b << 23); } // 2^(b - 127)+ static inline ushort ds4_f2bf16(float f) {+ const uint u = as_type<uint>(f);+ return ushort((u + 0x7fffu + ((u >> 16) & 1u)) >> 16);+ }+++ // ds4_kernels.cu:1104 ext_to_ring_kernel - the window's BF16 rows packed back into the E4M3 ring, each 32-value group at the+ // smallest power-of-two scale it fits under (k), so v / 2^k is an E4M3 value times a power of two.+ static inline uint ds4_e4m3_code(float v, int k) {+ const uint sign = (v < 0.0f) ? 0x80u : 0u;+ float a = fabs(v) * exp2(-float(k));+ if (a == 0.0f) return sign;+ int e = int(floor(log2(a)));+ int mant = int(round((a / exp2(float(e)) - 1.0f) * 8.0f));+ int ex = e + 7;+ if (mant >= 8) { mant = 0; ++ex; }+ if (ex <= 0) { mant = 0; ex = 0; }+ if (ex > 15) { mant = 7; ex = 15; }+ return sign | (uint(ex) << 3) | uint(mant & 7);+ }++ kernel void ds4_ext_to_ring(device const ushort* ext [[buffer(0)]], device uchar* ring [[buffer(1)]],+ device uchar* backup [[buffer(2)]], constant long& p0 [[buffer(3)]],+ constant uint& n [[buffer(4)]], constant uint& T [[buffer(5)]],+ constant uint& d [[buffer(6)]], constant ulong& rb [[buffer(7)]], constant uint& has_backup [[buffer(8)]],+ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ if (T - t > n) return; // overwritten by a later row of the window anyway+ device uchar* row = ring + (size_t) ((size_t(p0) + t) % n) * rb;+ if (has_backup != 0u) { // (a bound buffer is never a null pointer here: the host says whether there is a backup)+ for (size_t i = tid; i < rb; i += 256u) backup[(size_t) t * rb + i] = row[i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ const device ushort* src = ext + (size_t) (n + t) * d;+ for (uint e0 = 0; e0 < d; e0 += 256u) {+ const uint e = e0 + tid;+ const float v = e < d ? as_type<float>(uint(src[e]) << 16) : 0.0f; // (BF16 bits, not half)+ uint a = as_type<uint>(v) & 0x7fffffffu; // magnitudes ordered as bits+ for (uint o = 16u; o > 0u; o >>= 1u) a = max(a, simd_shuffle_xor(a, o));+ const int k = a == 0u ? 0 : int(a >> 23) - 135 + (((a & 0x7fffffu) > 0x600000u) ? 1 : 0);+ if (e >= d) continue;+ row[e] = uchar(ds4_e4m3_code(v, k));+ if ((e & 31u) == 0u) row[d + e / 32u] = uchar(k + 127);+ }+ }+++ // ds4_kernels.cu:916 unpack32 - 16 threads expand one MXFP4 KV row (288 B at d=512) into 32 BF16 values a thread:+ // 16 code bytes at part*16, two E4M3 scale bytes at 256 + part*2, and the 4-bit code is itself an FP4 bit pattern+ // (e2m1_val14: sign and exponent/mantissa bits spliced into a half).+ static inline float ds4_e2m1_val14(uint c) {+ const ushort h = (ushort) (((c & 8u) << 12) | ((c & 7u) << 9));+ return float(as_type<half>(h));+ }++ // ds4_kernels.cu:937 unpack_sel_kernel - the selected rows of the compressed cache into the attention scratch, one row a+ // group of 16 threads. n_sel comes either from the generated schedule (gen: min(kcap, (pos+1)/R)) or from nsel[t].+ kernel void ds4_unpack_sel(device const uchar* ckv [[buffer(0)]], device const int* sel [[buffer(1)]],+ constant uint& sel_stride [[buffer(2)]], constant int& gen [[buffer(3)]],+ constant long& p0 [[buffer(4)]], constant int& t0 [[buffer(5)]],+ constant int& R [[buffer(6)]], constant int& kcap [[buffer(7)]],+ device const int* nsel [[buffer(8)]], constant uint& cap [[buffer(9)]],+ device ushort* out [[buffer(10)]], constant ulong& rb [[buffer(11)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint t = tgp.y;+ int n_sel;+ if (gen != 0) {+ const long pa = p0 + long(t0) + long(t);+ n_sel = R > 0 ? int(min(long(kcap), (pa + 1) / long(R))) : 0;+ } else {+ n_sel = nsel[t];+ }+ const uint i = tgp.x * 16u + (tid >> 4u), part = tid & 15u;+ if (int(i) >= n_sel) return;+ const size_t r = (size_t) t * cap + i;+ const device uchar* rr = ckv + (size_t) sel[(size_t) t * sel_stride + i] * rb;+ const uint4 cw = *(const device uint4*) (rr + part * 16u);+ const float k0 = ds4_e4m3(uint(rr[256u + part * 2u])) * 16384.0f;+ const float k1 = ds4_e4m3(uint(rr[256u + part * 2u + 1u])) * 16384.0f;+ const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};+ uint o[16];+ for (uint k = 0; k < 16u; ++k) {+ const uint b = wv[k >> 2] >> (8u * (k & 3u));+ const float ks = k < 8u ? k0 : k1;+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);+ }+ device uint* dst = (device uint*) (out + r * 512u + part * 32u);+ for (uint q = 0; q < 4u; ++q) {+ dst[q * 4u + 0u] = o[4u * q + 0u];+ dst[q * 4u + 1u] = o[4u * q + 1u];+ dst[q * 4u + 2u] = o[4u * q + 2u];+ dst[q * 4u + 3u] = o[4u * q + 3u];+ }+ }+++ // ds4_kernels.cu:1772 moe_combine_kernel - a token's expert outputs back into its row: for each pick whose routed row is in+ // [r0, r1), acc += w[t][i] * y[r - r0][e]; untouched when no pick falls in range.+ kernel void ds4_moe_combine(device float* acc [[buffer(0)]], device const float* y [[buffer(1)]],+ device const int* pos [[buffer(2)]], device const float* w [[buffer(3)]],+ constant int& kk [[buffer(4)]], constant int& r0 [[buffer(5)]],+ constant int& r1 [[buffer(6)]], constant uint& dim [[buffer(7)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + tid;+ if (e >= dim) return;+ const uint t = tgp.y;+ float a = acc[(size_t) t * dim + e];+ bool any = false;+ for (int i = 0; i < 8; ++i) {+ if (i >= kk) break;+ const int r = pos[(size_t) t * 8u + i];+ if (r < r0 || r >= r1) continue;+ a = fma(w[(size_t) t * 8u + i], y[(size_t) (r - r0) * dim + e], a);+ any = true;+ }+ if (any) acc[(size_t) t * dim + e] = a;+ }+++ // ds4_kernels.cu:362 hc_post_kernel - like hc_post_inplace, but reads a separate residual and writes a separate output:+ // out[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*res[t][j][e]+ kernel void ds4_hc_post_v(device const float* y [[buffer(0)]], device const float* res [[buffer(1)]],+ device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],+ device float* out [[buffer(4)]], constant uint& dim [[buffer(5)]],+ constant uint& hc [[buffer(6)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const uint i = tgp.x * 256u + tid;+ if (i >= dim * hc) return;+ const uint t = tgp.y, k = i / dim, e = i - k * dim;+ const device float* r = res + (size_t) t * hc * dim;+ float a = post[(size_t) t * 8u + k] * y[(size_t) t * dim + e];+ for (uint j = 0; j < 8u; ++j) {+ if (j >= hc) break;+ a += comb[(size_t) t * 64u + j * hc + k] * r[(size_t) j * dim + e];+ }+ out[(size_t) t * hc * dim + i] = a;+ }+++ // ds4_kernels.cu:441 gemv_f32_kernel - a warp a row of F32 weights: float4 loads of W[row] and x[t], four FMAs a step,+ // lane-strided over k/4, then the warp sum into y[t][row].+ kernel void ds4_gemv_f32(device const float* W [[buffer(0)]], device const float* x [[buffer(1)]],+ constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],+ device float* y [[buffer(4)]], constant uint& ldy [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]]) {+ const uint row = tgp.x * 8u + (tid >> 5);+ if (row >= n) return;+ const uint t = tgp.y;+ const device float4* w = (const device float4*) (W + (size_t) row * kk);+ const device float4* xv = (const device float4*) (x + (size_t) t * kk);+ float acc = 0.0f;+ for (uint c = lane; c < kk / 4u; c += 32u) {+ const float4 a = w[c], b = xv[c];+ acc = fma(a.x, b.x, acc);+ acc = fma(a.y, b.y, acc);+ acc = fma(a.z, b.z, acc);+ acc = fma(a.w, b.w, acc);+ }+ acc += simd_shuffle_xor(acc, 16);+ acc += simd_shuffle_xor(acc, 8);+ acc += simd_shuffle_xor(acc, 4);+ acc += simd_shuffle_xor(acc, 2);+ acc += simd_shuffle_xor(acc, 1);+ if (lane == 0u) y[(size_t) t * ldy + row] = acc;+ }+++ // ds4_kernels.cu:1353 route_publish_kernel - the MoE router: sc = sqrt(softplus(logit)), bs = sc + bias, then k rounds of a+ // block-wide argmax (a strided max, a warp reduction, a 16-warp pass), the picks' sc summed into the weights, and each pick's+ // device address published to plan_ptr. The prefetch requests (req) are NOT published here yet: that is the schedule, not the+ // answer, and it can be turned on separately.+ kernel void ds4_route_publish(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],+ constant uint& n [[buffer(2)]], constant uint& kk [[buffer(3)]],+ constant float& route_scale [[buffer(4)]], device const ulong* vram [[buffer(5)]],+ device int* idx [[buffer(6)]], device float* w [[buffer(7)]],+ device ulong* plan_ptr [[buffer(8)]], device const float* bias_vl [[buffer(9)]],+ constant uint& vl [[buffer(10)]], constant uint& has_vl [[buffer(11)]],+ uint t [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ threadgroup float sc[1024];+ threadgroup float bs[1024];+ threadgroup float bv[16];+ threadgroup int bi[16];+ threadgroup int pick[8];+ const device float* lg = logits + (size_t) t * 1024u;+ const device float* b = (has_vl != 0u && ((vl >> t) & 1u) != 0u) ? bias_vl : bias;+ for (uint i = tid; i < n; i += 512u) {+ const float l = lg[i];+ const float sp = l > 20.0f ? l : log(1.0f + exp(l));+ sc[i] = sqrt(sp);+ bs[i] = sc[i] + b[i];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const uint lane = tid & 31u, wp = tid >> 5u;+ for (uint r = 0; r < 8u; ++r) {+ if (r >= kk) break;+ float best = -3.402823466e+38f;+ int bix = 0x7fffffff;+ for (uint i = tid; i < n; i += 512u) {+ if (bs[i] > best || (bs[i] == best && int(i) < bix)) { best = bs[i]; bix = int(i); }+ }+ for (uint o = 16u; o > 0u; o >>= 1u) {+ const float ov = simd_shuffle_xor(best, (ushort) o);+ const int oi = simd_shuffle_xor(bix, (ushort) o);+ if (ov > best || (ov == best && oi < bix)) { best = ov; bix = oi; }+ }+ if (lane == 0u && wp < 16u) { bv[wp] = best; bi[wp] = bix; }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid == 0u) {+ float bb = -3.402823466e+38f;+ int ix = 0x7fffffff;+ for (uint q = 0; q < 16u; ++q) {+ if (bv[q] > bb || (bv[q] == bb && bi[q] < ix)) { bb = bv[q]; ix = bi[q]; }+ }+ pick[r] = ix;+ bs[ix] = -3.402823466e+38f;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ if (tid == 0u) {+ float sum = 0.0f;+ for (uint r = 0; r < 8u; ++r) { if (r >= kk) break; sum += sc[pick[r]]; }+ for (uint r = 0; r < 8u; ++r) {+ if (r >= kk) break;+ const uint p_ = uint(pick[r]);+ const float wv = sc[p_] / (sum + 1e-20f) * route_scale;+ idx[(size_t) t * 8u + r] = int(p_);+ w[(size_t) t * 8u + r] = wv;+ plan_ptr[(size_t) t * 8u + r] = vram[p_];+ }+ }+ }+++ // ds4_kernels.cu swiglu_kernel - the expert MLP's activation: h = silu(g) * u, with the clamp when lim > 0+ kernel void ds4_swiglu(device const float* g [[buffer(0)]], device const float* u [[buffer(1)]],+ constant uint& n [[buffer(2)]], constant float& lim [[buffer(3)]],+ device float* h [[buffer(4)]], uint gid [[thread_position_in_grid]]) {+ if (gid >= n) return;+ float gv = g[gid], uv = u[gid];+ if (lim > 0.0f) { // both inputs clamped, then the gate itself+ gv = fmin(gv, lim);+ uv = fmin(fmax(uv, -lim), lim);+ }+ h[gid] = (gv / (1.0f + exp(-gv))) * uv; // silu(g) * u+ }++ kernel void ds4_ring_to_ext(device const uchar* ring [[buffer(0)]], device ushort* ext [[buffer(1)]],+ constant long& p0 [[buffer(2)]], constant uint& n [[buffer(3)]],+ constant uint& d [[buffer(4)]], constant ulong& rb [[buffer(5)]],+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const long p = p0 - long(n) + long(r);+ const device uchar* row = ring + (size_t) (p >= 0 ? (p % long(n)) : 0) * rb;+ for (uint e = tid; e < d; e += 256u) {+ ext[(size_t) r * d + e] = p >= 0 ? ds4_f2bf16(ds4_e4m3(uint(row[e])) * ds4_ue8m0(uint(row[d + e / 32u]))) : ushort(0);+ }+ }++ // ---------------------------------------------------------------- ds4_gguf.cu:577 mx_combine_kernel+ //+ // A token's expert outputs summed in plan order (an expert off the device adds 0), the CPU lane's part added when its+ // flag is set, then the shared expert's - ds4_gguf.cu:169's combine(). Grid (n_embd / 128, T), 128 threads.+ struct Ds4Comb {+ device const ulong* plan_ptr; // [T][8]: nonzero when the device holds that expert+ device const float* plan_w; // [T][8]+ device const float* part; // [entries][n_embd]+ device const float* cpu_part; // [T][cpu_stride], or null+ device const int* cpu_flag; // [T], or null+ device const float* sh_out; // [T][n_embd], or null+ device float* out;+ long cpu_stride, out_stride;+ int k, n_embd;+ };++ kernel void ds4_mx_combine(constant Ds4Comb& a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const uint r = tgp.x * 128u + tid;+ if (int(r) >= a.n_embd) return;+ const int t = int(tgp.y);+ float m = 0.0f;+ for (int i = 0; i < a.k; ++i) { // kPicks <= 8+ const int e = t * 8 + i;+ const float pi = a.plan_ptr[e] != 0ul ? a.part[(size_t) e * (size_t) a.n_embd + r] : 0.0f;+ m = fma(a.plan_w[t * 8 + i], pi, m);+ }+ if (a.cpu_flag != nullptr && a.cpu_flag[t] != 0) m += a.cpu_part[(size_t) t * (size_t) a.cpu_stride + r];+ m += a.sh_out != nullptr ? a.sh_out[(size_t) t * (size_t) a.n_embd + r] : 0.0f;+ a.out[(size_t) t * (size_t) a.out_stride + r] = m;+ }++ // ---------------------------------------------------------------- ds4_kernels.cu:1286 engram_dequant_kernel / :1311 engram_gate_kernel+ //+ // The Engram tables' row decoder and gate (this model's tables are fmt 2: IQ4_NL, an 18-byte block of 32 values whose+ // nibbles index ds4_kIq4nl above).+ kernel void ds4_engram_dequant_rows(device const uchar* rows [[buffer(0)]], constant uint& hd [[buffer(1)]],+ device float* out [[buffer(2)]], constant int& fmt [[buffer(3)]],+ uint r [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ if (fmt == 2) {+ const device uchar* row = rows + (size_t) r * (size_t) hd;+ for (uint e = tid; e < hd; e += 256u) {+ const device uchar* b = row + (size_t) (e / 32u) * 18u;+ const uint q = b[2u + (e & 15u)];+ const int v = (e & 16u) ? int(q >> 4) : int(q & 15u);+ const ushort dh = ushort(b[0]) | (ushort(b[1]) << 8);+ out[(size_t) r * (size_t) hd + e] = float(as_type<half>(dh)) * float(ds4_kIq4nl[v]);+ }+ return;+ }+ // fmt 1 (Q6_K) and fmt 0 (E4M3 x scale) are not ported: NaN rather than a silent zero (as ds4_engram_dequant does)+ for (uint e = tid; e < hd; e += 256u) out[(size_t) r * (size_t) hd + e] = NAN;+ }++ // ds4_kernels.cu's block_sum for a 1024-thread threadgroup: a warp sum a warp, then every warp sums the 32 partials (the+ // same result in all of them, so no warp diverges at a barrier)+ static inline float ds4_block_sum32(float v, threadgroup float* red, ushort lane, ushort w) {+ v = ds4_warp_sum(v);+ if (lane == 0) red[w] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float t = lane < 32u ? red[lane] : 0.0f;+ t = ds4_warp_sum(t);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return t;+ }++ kernel void ds4_engram_gate(device float* h [[buffer(0)]], device const float* kv [[buffer(1)]],+ device const float* w [[buffer(2)]], constant uint& hc [[buffer(3)]],+ constant uint& dim [[buffer(4)]], constant float& eps [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort wg [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[32];+ const uint k = tgp.x, t = tgp.y;+ device float* hk = h + ((size_t) t * hc + (size_t) k) * (size_t) dim;+ const device float* kvt = kv + (size_t) t * (size_t) (hc + 1u) * (size_t) dim;+ const device float* key = kvt + (size_t) k * (size_t) dim;+ const device float* val = kvt + (size_t) hc * (size_t) dim;+ const device float* wk = w + (size_t) k * (size_t) dim;+ float sh = 0.0f, sk = 0.0f, sd = 0.0f;+ for (uint e = tid; e < dim; e += 1024u) {+ const float hv = hk[e], kve = key[e];+ sh += hv * hv;+ sk += kve * kve;+ sd += hv * wk[e] * kve;+ }+ sh = ds4_block_sum32(sh, red, lane, wg);+ sk = ds4_block_sum32(sk, red, lane, wg);+ sd = ds4_block_sum32(sd, red, lane, wg);+ const float rstd = rsqrt(sh / float(dim) + eps) * rsqrt(sk / float(dim) + eps);+ const float dot = sd * rstd * rsqrt(float(dim));+ const float sq = copysign(sqrt(fmax(fabs(dot), 1e-6f)), dot);+ const float gate = 1.0f / (1.0f + exp(-sq));+ for (uint e = tid; e < dim; e += 1024u) hk[e] = hk[e] + gate * val[e];+ }++ // ds4_gguf.cu:72 bf16_mv_kernel<NC, true> - a BF16 matrix's matvec against F32 activations (the engram WKV here, and the+ // BF16 dense matrices the jobs' f == 0 names). Same frame as ds4_mx_mv above: a warp a row, a lane strides the row; here a+ // lane 8 BF16 weights (16 bytes = one uint4) and 8 activations, so the row is walked in 256-value steps.+ // A pair's F32 is bit-exact - the low half zeroed - which is bf16_mv_kernel's __uint_as_float(w << 16).+ kernel void ds4_bf16_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MxJob& b = *(const device Ds4MxJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MxJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const device ushort* w = (const device ushort*) b.w + (size_t) row * b.ld;+ thread float acc[16]; // (the CUDA template's largest NC; ncol cuts the rest)+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;+ for (uint i = uint(lane) * 8u; i < uint(b.k); i += 256u) {+ const uint4 q = *(const device uint4*) (w + i);+ const float wf[8] = {as_type<float>(q.x << 16), as_type<float>(q.x & 0xffff0000u),+ as_type<float>(q.y << 16), as_type<float>(q.y & 0xffff0000u),+ as_type<float>(q.z << 16), as_type<float>(q.z & 0xffff0000u),+ as_type<float>(q.w << 16), as_type<float>(q.w & 0xffff0000u)};+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device float4* xp = (const device float4*) (b.x + c * b.xcs + (size_t) i);+ const float4 x0 = xp[0], x1 = xp[1];+ float a = acc[c];+ a = fma(wf[0], x0.x, a);+ a = fma(wf[1], x0.y, a);+ a = fma(wf[2], x0.z, a);+ a = fma(wf[3], x0.w, a);+ a = fma(wf[4], x1.x, a);+ a = fma(wf[5], x1.y, a);+ a = fma(wf[6], x1.z, a);+ a = fma(wf[7], x1.w, a);+ acc[c] = a;+ }+ }+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+ }++ // ---------------------------------------------------------------- ds4_kernels.cu:524 compress_kernel / :582 index_score_kernel+ //+ // The compressor (ratio > 1 layers): each token's kv / score row into its slot, and every ratio'th token pooled by a+ // per-value softmax over the slots, scaled by a row RMS and the norm weights. Grid (1), 512 threads (d <= 1024 values).+ static inline float ds4_block_sum_n(float v, threadgroup float* red, ushort lane, ushort w, uint nw) {+ v = ds4_warp_sum(v);+ if (lane == 0) red[w] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float t = lane < nw ? red[lane] : 0.0f;+ t = ds4_warp_sum(t); // (every warp sums the same partials: no divergence at a barrier)+ threadgroup_barrier(mem_flags::mem_threadgroup);+ return t;+ }++ kernel void ds4_compress(device const float* kv [[buffer(0)]], device const float* score [[buffer(1)]],+ device float* kv_state [[buffer(2)]], device float* score_state [[buffer(3)]],+ constant int& ratio [[buffer(4)]], constant int& p0 [[buffer(5)]], constant int& T [[buffer(6)]],+ constant uint& d [[buffer(7)]], device const float* wn [[buffer(8)]],+ constant float& eps [[buffer(9)]], device float* latent [[buffer(10)]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort wg [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[32];+ const float kfmax = -3.4028234663852886e+38f; // -FLT_MAX, as the CUDA writes it+ int g = 0;+ for (int t = 0; t < T; ++t) {+ const int slot = (p0 + t) % ratio;+ const bool pool = (p0 + t + 1) % ratio == 0;+ for (uint e = tid; e < d; e += 512u) {+ kv_state[(size_t) slot * d + e] = kv[(size_t) t * d + e];+ score_state[(size_t) slot * d + e] = score[(size_t) t * d + e];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (!pool) continue;+ float val[2] = {0.0f, 0.0f}; // (d <= 2 * 512)+ float ss = 0.0f;+ uint n = 0;+ for (uint e = tid; e < d; e += 512u, ++n) {+ float m = kfmax;+ for (int s = 0; s < ratio; ++s) m = fmax(m, score_state[(size_t) s * d + e]);+ float den = 0.0f;+ for (int s = 0; s < ratio; ++s) den += exp(score_state[(size_t) s * d + e] - m);+ float acc = 0.0f;+ for (int s = 0; s < ratio; ++s)+ acc += kv_state[(size_t) s * d + e] * (exp(score_state[(size_t) s * d + e] - m) / den);+ val[n] = acc;+ ss += acc * acc;+ }+ const float inv = rsqrt(ds4_block_sum_n(ss, red, lane, wg, 16u) / float(d) + eps);+ n = 0;+ for (uint e = tid; e < d; e += 512u, ++n) latent[(size_t) g * d + e] = val[n] * inv * wn[e];+ ++g;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ }++ // ---------------------------------------------------------------- ds4_kernels.cu:582 index_score_kernel<128, KF>+ //+ // A thread a position: its key row (kfmt 3: 64 bytes of E2M1 codes (value 2i in the low nibble) then 4 UE8M0 scales; kfmt+ // 0: BF16) against every head's query (a threadgroup tile), relu, then the heads' weighted sum. Grid (ceil(n / 256), T).+ struct Ds4Rows { // ds4_kernels.cuh:115 Rows (kWin = 16)+ int n[16];+ int k[16];+ int gen, R, kcap, ccap, blk;+ long p0;+ };++ static inline void ds4_rows_at(constant Ds4Rows& r, int t, thread int& n, thread int& k) {+ if (r.gen == 0) {+ n = r.n[t];+ k = r.k[t];+ return;+ }+ const int nn = int((r.p0 + t + 1) / r.R);+ const int cb = min(r.ccap, (nn + r.blk - 1) / r.blk);+ switch (r.gen) {+ case 1: n = nn; k = min(r.kcap, nn); break;+ case 2: n = nn; k = cb; break;+ case 3: n = cb * r.blk; k = min(r.kcap, nn); break;+ default: n = (nn + r.blk - 1) / r.blk; k = r.kcap; break;+ }+ }++ kernel void ds4_index_score(device const float* q [[buffer(0)]], device const float* wq [[buffer(1)]],+ device const uchar* kc [[buffer(2)]], device const int* cand [[buffer(3)]],+ device float* out [[buffer(4)]], constant Ds4Rows& lim [[buffer(5)]],+ constant int& heads [[buffer(6)]], constant int& n [[buffer(7)]],+ constant int& cand_stride [[buffer(8)]], constant int& blk [[buffer(9)]],+ constant long& out_stride [[buffer(10)]], constant int& kfmt [[buffer(11)]],+ constant int& has_cand [[buffer(12)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ constexpr int HD = 128;+ threadgroup float qs[32 * HD + 32];+ const int t = int(tgp.y);+ for (uint i = tid; i < uint(heads) * uint(HD); i += 256u) qs[i] = q[(size_t) t * (size_t) heads * HD + i];+ for (uint i = tid; i < uint(heads); i += 256u) qs[uint(heads) * HD + i] = wq[(size_t) t * 64 + i];+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int i = int(tgp.x) * 256 + int(tid);+ if (i >= n) return;+ device float* o = out + (long) t * out_stride;+ int ln, lk;+ ds4_rows_at(lim, t, ln, lk);+ int p = i;+ if (has_cand != 0) {+ if (i >= lk * blk) {+ o[i] = -INFINITY;+ return;+ }+ p = cand[(size_t) t * (size_t) cand_stride + (size_t) (i / blk)] * blk + i % blk;+ }+ if (p >= ln) {+ o[i] = -INFINITY;+ return;+ }+ const size_t kb = kfmt == 3 ? (size_t) ((HD / 2 + HD / 32 + 15) / 16 * 16) : (size_t) HD * 2;+ const device uchar* kr = kc + (size_t) p * kb;+ float acc[32];+ for (uint h = 0; h < 32u; ++h) acc[h] = 0.0f;+ for (int c = 0; c < HD; c += 8) {+ float kvv[8];+ if (kfmt == 3) { // 8 codes in 4 bytes, the group's power-of-two scale (2^(b - 127))+ const uint cw = uint(kr[c / 2]) | (uint(kr[c / 2 + 1]) << 8) | (uint(kr[c / 2 + 2]) << 16) |+ (uint(kr[c / 2 + 3]) << 24);+ const float ks = as_type<float>(uint(kr[HD / 2 + c / 32]) << 23);+ for (uint x = 0; x < 8u; ++x) kvv[x] = ds4_e2m1_val14(cw >> (4 * x)) * 16384.0f * ks;+ } else {+ const device ushort* b = (const device ushort*) (kr + (size_t) c * 2);+ for (uint x = 0; x < 8u; ++x) kvv[x] = as_type<float>(uint(b[x]) << 16);+ }+ for (uint h = 0; h < 32u; ++h) {+ if (int(h) >= heads) break;+ threadgroup const float* qh = qs + h * HD + c;+ for (uint x = 0; x < 8u; ++x) acc[h] += qh[x] * kvv[x];+ }+ }+ float s = 0.0f;+ for (uint h = 0; h < 32u; ++h) {+ if (int(h) >= heads) break;+ s += fmax(acc[h], 0.0f) * qs[uint(heads) * HD + h];+ }+ o[i] = s;+ }+++// The sparse-attention indexer with the GPU's matrix units. The scalar ds4_index_score spends 16.3 s of a 32K prompt++// (30 % of it) doing 32 head-dots a (query, key) pair one FMA at a time; this stages 16 fp4 keys and one query's q as++// halves and runs the 32 x 128 x 16 product through simdgroup_matrix, which measured 3.17-6.10x faster in the standalone++// harness (tools/ds4/idx_harness/idx_mma) with every output matching the scalar path to half precision (<= 7e-4++// relative). Same arithmetic, same order per (query, key) up to that rounding.++//++// The fp4 weights are staged as e2m1 * ks (<= 12, half-safe) with the 16384 applied once in the epilogue - the full++// 16384 * ks reaches 2e5 and overflows half.++kernel void ds4_index_score_mma(device const float* q [[buffer(0)]], device const float* wq [[buffer(1)]],++ device const uchar* kc [[buffer(2)]], device float* out [[buffer(4)]],++ constant Ds4Rows& lim [[buffer(5)]], constant int& heads [[buffer(6)]],++ constant int& n [[buffer(7)]], constant int& T [[buffer(8)]],++ constant long& out_stride [[buffer(9)]],++ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],++ uint sg [[simdgroup_index_in_threadgroup]]) {++ constexpr int HD = 128;++ constexpr int KB = 80; // the fp4 row: 64 code bytes + 4 scale bytes, padded to 80++ constexpr int NK = 16; // keys a threadgroup scores (two key-blocks of 8)++ threadgroup half qst[32 * HD]; // [head][dim]++ threadgroup half kst[HD * NK]; // [dim][key], e2m1 * ks++ threadgroup float part[4][NK]; // the four head-blocks' partial sums++ threadgroup float accs[8][64]; // a simdgroup's 8x8 tile++ const int k0 = int(tgp.x) * NK;++ const int t = int(tgp.y);++ if (t >= T) return;++ for (uint i = tid; i < uint(heads) * uint(HD); i += 256u) qst[i] = half(q[(size_t) t * (size_t) heads * HD + i]);++ for (int key = int(tid) / 8; key < NK; key += 32) {++ const int idx = k0 + key;++ if (idx >= n) continue;++ const device uchar* kr = kc + (size_t) idx * (size_t) KB;++ for (int c = 0; c < HD; c += 8) {++ const uint cw = uint(kr[c / 2]) | (uint(kr[c / 2 + 1]) << 8) | (uint(kr[c / 2 + 2]) << 16) |++ (uint(kr[c / 2 + 3]) << 24);++ const float ks = as_type<float>(uint(kr[HD / 2 + c / 32]) << 23);++ for (uint x = 0; x < 8u; ++x) kst[(c + int(x)) * NK + key] = half(ds4_e2m1_val14(cw >> (4 * x)) * ks);++ }++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ const int hb = int(sg) / 2, kb = int(sg) % 2; // eight heads x eight keys a simdgroup++ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);++ for (int c = 0; c < HD; c += 8) {++ simdgroup_half8x8 a, b;++ simdgroup_load(a, qst, HD, ulong2(ulong(c), ulong(hb) * 8ul));++ simdgroup_load(b, kst, NK, ulong2(ulong(kb) * 8ul, ulong(c)));++ simdgroup_multiply_accumulate(acc, a, b, acc);++ }++ simdgroup_store(acc, accs[sg], 8);++ simdgroup_barrier(mem_flags::mem_threadgroup);++ const uint lane = tid % 32u;++ if (lane < 8u) {++ const int key = kb * 8 + int(lane);++ float ssum = 0.0f;++ for (uint h = 0; h < 8u; ++h) {++ const int hh = hb * 8 + int(h);++ if (hh >= heads) break;++ ssum += fmax(accs[sg][h * 8 + lane] * 16384.0f, 0.0f) * wq[(size_t) t * 64 + hh];++ }++ part[hb][key] = ssum;++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tid < uint(NK)) {++ const int key = int(tid);++ const int idx = k0 + key;++ if (idx >= n) return;++ int ln, lk;++ ds4_rows_at(lim, t, ln, lk);++ device float* o = out + (long) t * out_stride;++ o[idx] = idx >= ln ? -INFINITY : (part[0][key] + part[1][key] + part[2][key] + part[3][key]);++ }++}+++ // ---------------------------------------------------------------- ds4_kernels.cu:142 finish_kernel<true> (the KV pack)+ //+ // The same load, RMS and mrope as ds4_finish above, then the row written as the cache's packed codes (kv_row_bytes): the+ // quantiser's scale byte a group (32 values, or 16 for the 4-bit MXFP4-style rows), the codes themselves from bit 0 up -+ // for the 4-bit formats two values a byte, the second from the next lane. Written from the CUDA of the same name.++++ static inline uint ds4_e2m1_code(float q) { // (bit 3 the sign, -0 kept; the magnitudes as 0 .. 7)+ const float a = fabs(q);+ const uint m = a <= 1.5f ? uint(a * 2.0f) : a <= 2.0f ? 4u : a <= 3.0f ? 5u : a <= 4.0f ? 6u : 7u;+ return m | ((as_type<uint>(q) >> 31) << 3);+ }++ kernel void ds4_finish_pack(device float* x [[buffer(0)]], constant long& stride [[buffer(1)]],+ constant uint& len [[buffer(2)]], constant Ds4Finish& f [[buffer(3)]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]], uint b [[threadgroup_position_in_grid]]) {+ threadgroup float red[8];+ device float* v = x + (long) b * stride;+ const uint nj = (len + 255u) / 256u;+ float r[32];+ float ss = 0.0f;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ r[j] = e < len ? v[e] : 0.0f;+ ss += r[j] * r[j];+ }+ if (f.w != nullptr) {+ const float tot = ds4_block_sum(ss, red, lane, w);+ const float inv = rsqrt(tot / float(len) + f.eps);+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ if (e < len) r[j] = r[j] * inv * f.w[e];+ }+ }+ if (f.rd > 0) {+ const uint r0 = len - uint(f.rd);+ const uint pos = uint(f.pos) + (b / uint(f.vpp)) * uint(f.pos_step);+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ const float other = simd_shuffle_xor(r[j], 1);+ if (e >= r0 && e < len) {+ const uint i = (e - r0) >> 1;+ const float ang = float(pos) * f.freqs[i];+ const float c = cos(ang);+ const float sn = f.inverse ? -sin(ang) : sin(ang);+ r[j] = ((e - r0) & 1u) == 0u ? r[j] * c - other * sn : other * sn + r[j] * c;+ }+ }+ }+ // the packed row: a group's scale and the group's codes as the format's (quant 1: d E4M3 codes at pr[e] and d/32+ // scale bytes at pr[len + e/32]; 2: E2M1 pairs at pr[e/2] with E4M3 scales at pr[len/2 + e/16]; 3: E2M1 pairs with+ // UE8M0 scales at pr[len/2 + e/32])+ const uint g = f.quant == 2 ? 16u : 32u;+ device uchar* pr = f.pdst + (long) b * f.pdst_stride;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ float a = e < len ? fabs(r[j]) : 0.0f;+ for (uint o = g / 2u; o > 0u; o >>= 1) a = fmax(a, simd_shuffle_xor(a, ushort(o)));+ float q = 0.0f, s = 1.0f;+ if (e < len) {+ if (f.quant == 1) {+ s = ds4_pow2_ceil(fmax(a, 1e-4f) * (1.0f / 448.0f));+ q = ds4_e4m3_round(fmin(fmax(r[j] / s, -448.0f), 448.0f));+ } else if (f.quant == 2) {+ s = ds4_e4m3_round(fmax(a, 6.0f * 0.001953125f) / 6.0f);+ q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));+ } else {+ s = ds4_pow2_ceil(fmax(a, 6.0f * 1.1754943508222875e-38f) * (1.0f / 6.0f));+ q = ds4_e2m1_rt(fmin(fmax(r[j] / s, -6.0f), 6.0f));+ }+ r[j] = q * s;+ }+ if (f.quant == 1) {+ if (e < len) pr[e] = uchar(ds4_e4m3_code(q, 0));+ } else {+ const uint c = ds4_e2m1_code(q);+ const uint hi = simd_shuffle_down(c, 1); // (value e + 1: the next lane's)+ if (e < len && (e & 1u) == 0u) pr[e >> 1] = uchar(c | (hi << 4));+ }+ if (e < len && e % g == 0u)+ pr[(f.quant == 1 ? len : len / 2u) + e / g] =+ uchar(f.quant == 2 ? ds4_e4m3_code(s, 0) : (as_type<uint>(s) >> 23));+ }+ device ushort* d = f.dst != nullptr ? f.dst + (long) b * f.dst_stride : nullptr;+ for (uint j = 0; j < 32u; ++j) {+ if (j >= nj) break;+ const uint e = tid + j * 256u;+ if (e >= len) continue;+ if (d != nullptr) {+ const uint bits = as_type<uint>(r[j]);+ d[e] = ushort((bits + 0x7fffu + ((bits >> 16) & 1u)) >> 16); // BF16 round to nearest even+ }+ }+ }++ // ds4_kernels.cu:401 gemv_f16_rows_kernel - F16 weights against F32 activations (the indexer's wproj here): a warp a row+ // of the matrix, a lane 8 halves (16 bytes, the uint4 above) and 8 activations a step, kRowsT = 2 tokens a block. Grid+ // (ceil(n / 8), ceil(T / 2)), threadgroup 256.+ kernel void ds4_gemv_f16_rows(device const ushort* W [[buffer(0)]], device const float* x [[buffer(1)]],+ constant ulong& ldx [[buffer(2)]], constant int& n [[buffer(3)]],+ constant int& k [[buffer(4)]], constant int& T [[buffer(5)]],+ constant float& alpha [[buffer(6)]], device float* y [[buffer(7)]],+ constant ulong& ldy [[buffer(8)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ constexpr int kRowsT = 2;+ const int row = int(tgp.x) * 8 + int(tid >> 5);+ const int t0 = int(tgp.y) * kRowsT, nt = min(kRowsT, T - t0);+ if (row >= n) return;+ const device ushort* w = W + (size_t) row * (size_t) k;+ float acc[kRowsT];+ for (int t = 0; t < kRowsT; ++t) acc[t] = 0.0f;+ for (int c = int(lane) * 8; c < k; c += 32 * 8) {+ const uint4 q = *(const device uint4*) (w + c);+ const float2 f0 = float2(as_type<half2>(q.x)), f1 = float2(as_type<half2>(q.y));+ const float2 f2 = float2(as_type<half2>(q.z)), f3 = float2(as_type<half2>(q.w));+ const float2 fs[4] = {f0, f1, f2, f3};+ float xv[kRowsT][8];+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) {+ const device float4* xp = (const device float4*) (x + (size_t) (t0 + t) * ldx + (size_t) c);+ const float4 a4 = xp[0], b4 = xp[1];+ xv[t][0] = a4.x;+ xv[t][1] = a4.y;+ xv[t][2] = a4.z;+ xv[t][3] = a4.w;+ xv[t][4] = b4.x;+ xv[t][5] = b4.y;+ xv[t][6] = b4.z;+ xv[t][7] = b4.w;+ }+ for (int i = 0; i < 4; ++i) {+ const float2 f = fs[i];+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) acc[t] += f.x * xv[t][2 * i] + f.y * xv[t][2 * i + 1];+ }+ }+ for (int t = 0; t < kRowsT; ++t)+ if (t < nt) {+ float a = acc[t];+ a += simd_shuffle_xor(a, 16);+ a += simd_shuffle_xor(a, 8);+ a += simd_shuffle_xor(a, 4);+ a += simd_shuffle_xor(a, 2);+ a += simd_shuffle_xor(a, 1);+ if (lane == 0u) y[(size_t) (t0 + t) * ldy + row] = alpha * a;+ }+ }++ // ---------------------------------------------------------------- ds4_kernels.cu:650 topk_kernel+ //+ // A token's k largest scores: four 8-bit radix passes over the order keys of the row (a threadgroup of 1024, shared+ // histograms), then the picks written in index order by a scan a pass, so the attention sums them in the same order every+ // run and the ties kept are the lowest positions.+ static inline uint ds4_ord_key(float f) {+ const uint b = as_type<uint>(f);+ return (b & 0x80000000u) ? ~b : (b | 0x80000000u);+ }++ kernel void ds4_topk(device const float* vals [[buffer(0)]], constant long& stride [[buffer(1)]],+ constant Ds4Rows& r [[buffer(2)]], device const int* cand [[buffer(3)]],+ constant int& cand_stride [[buffer(4)]], constant int& blk [[buffer(5)]],+ device int* out [[buffer(6)]], constant int& out_stride [[buffer(7)]],+ constant int& has_cand [[buffer(8)]], uint tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort wid [[simdgroup_index_in_threadgroup]]) {+ threadgroup uint hist[256];+ threadgroup uint prefix_s, mask_s;+ threadgroup int remaining_s, cnt_gt, cnt_eq;+ threadgroup int wgt[32], weq[32];+ const int t = int(tgp);+ int n, k;+ ds4_rows_at(r, t, n, k);+ device const float* v = vals + (long) t * stride;+ device int* o = out + (size_t) t * (size_t) out_stride;+ device const int* cd = has_cand != 0 ? cand + (size_t) t * (size_t) cand_stride : nullptr;+ if (n <= 0) return;+ if (k >= n) {+ for (uint i = tid; i < uint(n); i += 1024u)+ o[i] = cd != nullptr ? cd[i / uint(blk)] * blk + int(i % uint(blk)) : int(i);+ return;+ }+ if (tid == 0u) {+ prefix_s = 0u;+ mask_s = 0u;+ remaining_s = k;+ cnt_gt = 0;+ cnt_eq = 0;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int shift = 24; shift >= 0; shift -= 8) {+ for (uint i = tid; i < 256u; i += 1024u) hist[i] = 0u;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const uint prefix = prefix_s, mask = mask_s;+ for (int i = int(tid); i < n; i += 1024) {+ const uint u = ds4_ord_key(v[i]);+ if ((u & mask) == prefix) {+ threadgroup atomic_uint* hp = (threadgroup atomic_uint*) &hist[(u >> shift) & 0xffu];+ atomic_fetch_add_explicit(hp, 1u, memory_order_relaxed);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid == 0u) {+ const int rem = remaining_s;+ int cum = 0;+ for (int d = 255; d >= 0; --d) {+ const int h = int(hist[d]);+ if (cum + h >= rem) {+ prefix_s = prefix | (uint(d) << shift);+ mask_s = mask | (0xffu << shift);+ remaining_s = rem - cum;+ break;+ }+ cum += h;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ const uint thr = prefix_s;+ const int remaining = remaining_s;+ const int n_gt = k - remaining;+ for (int i0 = 0; i0 < n; i0 += 1024) {+ const int i = i0 + int(tid);+ const uint u = i < n ? ds4_ord_key(v[i]) : 0u;+ const bool gt = i < n && u > thr, eq = i < n && u == thr;+ const auto bgv = simd_ballot(gt);+ const auto bev = simd_ballot(eq);+ const uint bg = uint((simd_vote::vote_t)bgv), be = uint((simd_vote::vote_t)bev);+ if (lane == 0u) {+ wgt[wid] = int(popcount(bg));+ weq[wid] = int(popcount(be));+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (wid == 0u) {+ const int g0 = wgt[lane], e0 = weq[lane];+ int gs = g0, es = e0;+ for (uint o2 = 1u; o2 < 32u; o2 <<= 1) {+ const int a = simd_shuffle_up(gs, ushort(o2)), b = simd_shuffle_up(es, ushort(o2));+ if (uint(lane) >= o2) {+ gs += a;+ es += b;+ }+ }+ wgt[lane] = cnt_gt + gs - g0;+ weq[lane] = cnt_eq + es - e0;+ simdgroup_barrier(mem_flags::mem_none);+ if (lane == 31u) {+ cnt_gt += gs;+ cnt_eq += es;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ int slot = -1;+ const uint below = (1u << lane) - 1u;+ if (gt) slot = wgt[wid] + int(popcount(bg & below));+ else if (eq) {+ const int e = weq[wid] + int(popcount(be & below));+ if (e < remaining) slot = n_gt + e;+ }+ if (slot >= 0) o[slot] = cd != nullptr ? cd[i / blk] * blk + i % blk : i;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ }++ // ds4_kernels.cu:751 block_max_kernel - a block of `blk` positions' largest score (the indexer's candidate blocks), the+ // newest position's block pinned in at +inf. Grid (ceil(nb_max / 256), T), 256 threads.+ kernel void ds4_block_max(device const float* score [[buffer(0)]], constant long& stride [[buffer(1)]],+ constant Ds4Rows& r [[buffer(2)]], constant int& blk [[buffer(3)]],+ device float* bm [[buffer(4)]], constant long& bm_stride [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const int b = int(tgp.x) * 256 + int(tid), t = int(tgp.y);+ int n, k_;+ ds4_rows_at(r, t, n, k_);+ const int nb = (n + blk - 1) / blk;+ if (b >= nb) return;+ float m = -INFINITY;+ for (int j = 0; j < blk; ++j) {+ const int p = b * blk + j;+ if (p < n) m = fmax(m, score[(long) t * stride + p]);+ }+ if (b == (n - 1) / blk) m = INFINITY; // the newest position's block is pinned in+ bm[(long) t * bm_stride + b] = m;+ }++ // ---------------------------------------------------------------- ds4_kernels.cu:1141 hc_mean_kernel / :1150 iota_rows_kernel+ //+ // The two small shapes the decode loop asks for: a token's mean over its hc streams, and a row's indices.+ kernel void ds4_hc_mean(device const float* h [[buffer(0)]], constant int& hc [[buffer(1)]],+ constant int& dim [[buffer(2)]], device float* out [[buffer(3)]],+ constant long& out_stride [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const int e = int(tgp.x) * 256 + int(tid), t = int(tgp.y);+ if (e >= dim) return;+ float a = 0.0f;+ for (int s = 0; s < hc; ++s) a += h[((size_t) t * (size_t) hc + (size_t) s) * (size_t) dim + (size_t) e];+ out[(long) t * out_stride + e] = a / float(hc);+ }++ kernel void ds4_iota_rows(device int* out [[buffer(0)]], constant int& n [[buffer(1)]], constant int& stride [[buffer(2)]],+ uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint t = tg;+ for (uint i = tid; i < uint(n); i += 64u) out[(size_t) t * (size_t) stride + i] = int(i);+ }++ // ---------------------------------------------------------------- vendor/sampler.cu:93 sampler_greedy_kernel+ //+ // The plain argmax over the vocabulary, ONE BLOCK (1024) a token: a thread's strided candidates with the repetition+ // penalties applied (a threadgroup bitmap of the history window says which tokens can be penalised at all), a warp+ // reduction, then warp 0's - a tie keeps the larger value and the SMALLER index, so this is the serial scan's answer.+ kernel void ds4_sampler_greedy(device const float* logits [[buffer(0)]], constant int& n_vocab [[buffer(1)]],+ device const int* history [[buffer(2)]], constant int& history_len [[buffer(3)]],+ constant int& plen [[buffer(4)]], constant float& pen_repeat [[buffer(5)]],+ constant float& pen_freq [[buffer(6)]], constant float& pen_present [[buffer(7)]],+ device const float* logit_bias [[buffer(8)]], constant int& has_bias [[buffer(9)]],+ device int* out [[buffer(10)]], uint tg [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort wid [[simdgroup_index_in_threadgroup]]) {+ threadgroup uint penal_bits[4096]; // (n_vocab + 31) / 32 words (the host checks it fits)+ threadgroup float sv[32];+ threadgroup int si[32];+ const int t = int(tg);+ device const float* l = logits + (size_t) t * (size_t) n_vocab;+ device const int* hrow = history != nullptr ? history + (size_t) t * (size_t) history_len : nullptr;+ int hlen = 0;+ if (hrow != nullptr) {+ hlen = min(plen, history_len);+ if (hlen < 0) hlen = 0;+ hrow += history_len - hlen; // the window is the tail+ }+ const int bits_words = (n_vocab + 31) / 32;+ const bool use_bits = hrow != nullptr && hlen > 0 && bits_words > 0;+ if (use_bits) {+ for (int w = int(tid); w < bits_words; w += 1024) penal_bits[w] = 0u;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int i = int(tid); i < hlen; i += 1024)+ if (hrow[i] >= 0 && hrow[i] < n_vocab) {+ threadgroup atomic_uint* pb = (threadgroup atomic_uint*) &penal_bits[hrow[i] >> 5];+ atomic_fetch_or_explicit(pb, 1u << (hrow[i] & 31), memory_order_relaxed);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ float bv = -INFINITY;+ int best = n_vocab; // (the "no candidate" index: it loses every comparison)+ for (int v = int(tid); v < n_vocab; v += 1024) {+ float s = has_bias != 0 ? l[v] + logit_bias[v] : l[v];+ int cnt = 0;+ if (use_bits && (penal_bits[v >> 5] & (1u << (v & 31))) != 0u)+ for (int i = 0; i < hlen; ++i)+ if (hrow[i] == v) ++cnt;+ if (cnt > 0) {+ if (s <= 0.0f) s *= pen_repeat;+ else s /= pen_repeat;+ s -= float(cnt) * pen_freq + 1.0f * pen_present;+ }+ if (s > bv) {+ bv = s;+ best = v;+ }+ }+ for (uint o = 16u; o > 0u; o >>= 1) {+ const float ov = simd_shuffle_down(bv, ushort(o));+ const int oi = simd_shuffle_down(best, ushort(o));+ if (ov > bv || (ov == bv && oi < best)) {+ bv = ov;+ best = oi;+ }+ }+ if (lane == 0u) {+ sv[wid] = bv;+ si[wid] = best;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (wid == 0u) {+ float wv = sv[lane];+ int wi = si[lane];+ for (uint o = 16u; o > 0u; o >>= 1) {+ const float ov = simd_shuffle_down(wv, ushort(o));+ const int oi = simd_shuffle_down(wi, ushort(o));+ if (ov > wv || (ov == wv && oi < wi)) {+ wv = ov;+ wi = oi;+ }+ }+ if (lane == 0u) out[t] = (wi < n_vocab) ? wi : 0; // (a tie between two -inf candidates: 0)+ }+ }++ // ---------------------------------------------------------------- sparse attention (the prompt path)+ //+ // ds4_kernels.cu:954 sparse_attn_packed_kernel<G> and :774 sparse_attn_kernel<false>. The packed kernel stages sixteen+ // rows of a tile in threadgroup memory (the window rows as they are, the compressed rows decoded to BF16) once for all+ // G heads of a threadgroup, then does each head's dots, softmax and weighted sum. The BF16 kernel is the same arithmetic+ // for row sets already stored as BF16 (a DSpark block); one head and query a threadgroup.+ struct Ds4AttnRows { // ds4_kernels.cuh:149 AttnRows (kWin = 16)+ int win0[16], nwin[16], nsel[16];+ int gen, t0, win, R, kcap;+ long p0;+ };++ static inline void ds4_attn_counts(constant Ds4AttnRows& ar, int t, thread int& n_win, thread int& n_sel, thread int& w0) {+ if (ar.gen != 0) {+ const long ta = (long) ar.t0 + (long) t;+ const long pa = ar.p0 + ta;+ const long a = (long) ar.win - ar.p0, b = ta + 1;+ const long first = a > b ? a : b; // max(win - p0, ta + 1)+ w0 = int(first);+ n_win = int((long) ar.win + ta - first + 1);+ n_sel = ar.R > 0 ? int(((pa + 1) / (long) ar.R) < (long) ar.kcap ? (pa + 1) / (long) ar.R : (long) ar.kcap) : 0;+ } else {+ w0 = ar.win0[t];+ n_win = ar.nwin[t];+ n_sel = ar.nsel[t];+ }+ if (n_win < 0) n_win = 0;+ if (n_sel < 0) n_sel = 0;+ }++ static inline float ds4_bf16_u32(uint v) { return as_type<float>(v << 16); }++ // 32 values of a packed compressed row (kKvFp4, 512 values) as BF16 in four uint4s+ static inline void ds4_attn_unpack32(const device uchar* rr, int part, threadgroup uint4* dst) {+ const uint4 cw = *(const device uint4*) (rr + (size_t) part * 16u);+ const float k0 = ds4_e4m3(uint(rr[256u + uint(part) * 2u])) * 16384.0f;+ const float k1 = ds4_e4m3(uint(rr[256u + uint(part) * 2u + 1u])) * 16384.0f;+ const uint wv[4] = {cw.x, cw.y, cw.z, cw.w};+ uint o[16];+ for (uint k = 0; k < 16u; ++k) {+ const uint b = wv[k >> 2] >> (8u * (k & 3u));+ const float ks = k < 8u ? k0 : k1;+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);+ }+ dst[0] = uint4(o[0], o[1], o[2], o[3]);+ dst[1] = uint4(o[4], o[5], o[6], o[7]);+ dst[2] = uint4(o[8], o[9], o[10], o[11]);+ dst[3] = uint4(o[12], o[13], o[14], o[15]);+ }++ // rows [s0, s0 + n) of the tile at t0 into tile4; a thread stages one row's 32 values+ static inline void ds4_attn_stage(device const ushort* wbase, device const uchar* ckv, device const int* sl,+ int n_win, int t0, int s0, int n, int sr, int part, threadgroup uint4* tile4) {+ if (sr >= n) return;+ const int j = s0 + sr;+ threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) part * 4u;+ const int jj = t0 + j;+ if (jj < n_win) {+ const device uint4* src = (const device uint4*) (wbase + (size_t) jj * 512u + (size_t) part * 32u);+ dst[0] = src[0];+ dst[1] = src[1];+ dst[2] = src[2];+ dst[3] = src[3];+ return;+ }+ const device uchar* rr = ckv + (size_t) sl[jj - n_win] * 288u;+ ds4_attn_unpack32(rr, part, dst);+ }++ static inline float ds4_attn_warp_max(float v) {+ v = fmax(v, simd_shuffle_xor(v, 16));+ v = fmax(v, simd_shuffle_xor(v, 8));+ v = fmax(v, simd_shuffle_xor(v, 4));+ v = fmax(v, simd_shuffle_xor(v, 2));+ v = fmax(v, simd_shuffle_xor(v, 1));+ return v;+ }++ template <int G>+ static inline void ds4_sparse_packed_run(device const float* q, device const ushort* win, device const uchar* ckv,+ device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,+ device const float* sink, uint heads, float scale, device float* out,+ threadgroup uint4* tile4, threadgroup float* sc, threadgroup float* m_run_s,+ threadgroup float* tscale_s, threadgroup float* den_s, uint2 tgp, uint tid,+ ushort lane_in, ushort wp_in) {+ constexpr int W = 8 / G; // warps a head+ const int hb = int(tgp.x) * G, t = int(tgp.y);+ int n_win, n_sel, w0;+ ds4_attn_counts(ar, t, n_win, n_sel, w0);+ const device ushort* wbase = win + (size_t) w0 * 512u;+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;+ const uint lane = uint(lane_in), wp = uint(wp_in);+ const uint gw = wp / uint(W), wr = wp % uint(W);+ float qr[16];+ for (int x = 0; x < 16; ++x)+ qr[x] = q[((size_t) t * heads + uint(hb) + gw) * 512u + lane * 16u + uint(x)];+ if (tid < uint(G)) {+ m_run_s[tid] = -1e30f;+ den_s[tid] = 0.0f;+ }+ float o0[G], o1[G];+ for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.0f;+ const int c = int(tid) * 2;+ const int sr = int(tid) >> 4, part = int(tid) & 15;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int total = n_win + n_sel;+ for (int t0 = 0; t0 < total; t0 += 64) {+ const int nt = min(64, total - t0);+ for (int s0 = 0; s0 < nt; s0 += 16) { // the dots+ const int n = min(16, nt - s0);+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int r = int(wr); r < n; r += W) {+ const threadgroup uint4* rp = tile4 + (size_t) r * 64u + lane * 2u;+ const uint4 a = rp[0], b = rp[1];+ const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};+ float s = 0.0f;+ for (int x = 0; x < 4; ++x) {+ s += qr[2 * x] * ds4_bf16_u32(av[x] & 0xffffu);+ s += qr[2 * x + 1] * ds4_bf16_u32(av[x] >> 16);+ }+ for (int x = 0; x < 4; ++x) {+ s += qr[8 + 2 * x] * ds4_bf16_u32(bv[x] & 0xffffu);+ s += qr[8 + 2 * x + 1] * ds4_bf16_u32(bv[x] >> 16);+ }+ s = ds4_warp_sum(s);+ if (lane == 0u) sc[uint(gw) * 64u + uint(s0 + r)] = s * scale;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ if (wp < uint(G)) { // head wp's softmax over the tile+ threadgroup float* scg = sc + wp * 64u;+ float m = -INFINITY;+ for (int j = int(lane); j < nt; j += 32) m = fmax(m, scg[j]);+ m = ds4_attn_warp_max(m);+ const float m_old = m_run_s[wp];+ const float m_new = fmax(m_old, m);+ float s = 0.0f;+ for (int j = int(lane); j < nt; j += 32) {+ const float p = exp(scg[j] - m_new);+ scg[j] = p;+ s += p;+ }+ s = ds4_warp_sum(s);+ if (lane == 0u) {+ tscale_s[wp] = exp(m_old - m_new);+ den_s[wp] = den_s[wp] * tscale_s[wp] + s;+ m_run_s[wp] = m_new;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int g = 0; g < G; ++g) {+ const float cs = tscale_s[g];+ o0[g] *= cs;+ o1[g] *= cs;+ }+ const threadgroup uint* tu = (const threadgroup uint*) tile4;+ for (int s0 = 0; s0 < nt; s0 += 16) { // the weighted sums+ const int n = min(16, nt - s0);+ ds4_attn_stage(wbase, ckv, sl, n_win, t0, s0, n, sr, part, tile4);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int r = 0; r < n; ++r) {+ const uint pr = tu[(size_t) r * 256u + uint(tid)];+ const float v0 = ds4_bf16_u32(pr & 0xffffu), v1 = ds4_bf16_u32(pr >> 16);+ for (int g = 0; g < G; ++g) {+ const float p = sc[uint(g) * 64u + uint(s0 + r)];+ o0[g] = fma(p, v0, o0[g]);+ o1[g] = fma(p, v1, o1[g]);+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ }+ for (int g = 0; g < G; ++g) {+ const float den = den_s[g] + exp(sink[hb + g] - m_run_s[g]);+ device float* ov = out + ((size_t) t * heads + uint(hb + g)) * 512u;+ ov[c] = o0[g] / den;+ ov[c + 1] = o1[g] / den;+ }+ }++ #define DS4_SPARSE_PACKED_SIG(NAME) \+ kernel void NAME(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]], \+ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]], \+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]], \+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]], \+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]], \+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]], \+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]])++ #define DS4_SPARSE_PACKED_BODY(G) \+ { \+ threadgroup uint4 tile4[16u * 64u]; \+ threadgroup float sc[8u * 64u]; \+ threadgroup float m_run_s[8u], tscale_s[8u], den_s[8u]; \+ ds4_sparse_packed_run<G>(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, tile4, sc, m_run_s, \+ tscale_s, den_s, tgp, tid, lane, wp); \+ }++ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g8) DS4_SPARSE_PACKED_BODY(8)+ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g4) DS4_SPARSE_PACKED_BODY(4)+ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g2) DS4_SPARSE_PACKED_BODY(2)+ DS4_SPARSE_PACKED_SIG(ds4_sparse_attn_g1) DS4_SPARSE_PACKED_BODY(1)++ #undef DS4_SPARSE_PACKED_SIG+ #undef DS4_SPARSE_PACKED_BODY++ #ifndef DS4_MX_PLO-#define DS4_MX_PLO 1 // 1: the softmax weights as BF16 high + low (rms error vs the scalar kernel 2e-6 instead of 3e-4)++#define DS4_MX_PLO 0 // 1: the softmax weights as BF16 high + low (rms error vs the scalar kernel 2e-6 instead of 3e-4)+ #endif+ struct Ds4AttnRaw { uint4 v[4]; int kind; }; // a thread's 32 values of one row, as read: kind 0 window (v[0..3] BF16), 1 compressed (v[0] codes, v[1].x the two scale bytes), 2 none++ static inline void ds4_attn_ld(device const ushort* wbase, device const uchar* ckv, device const int* sl, int n_win, int t0, int n,+ int sr, int pt, thread Ds4AttnRaw& r) {+ r.kind = 2;+ if (sr >= n) return;+ const int jj = t0 + sr;+ if (jj < n_win) {+ const device uint4* src = (const device uint4*) (wbase + (size_t) jj * 512u + (size_t) pt * 32u);+ r.v[0] = src[0]; r.v[1] = src[1]; r.v[2] = src[2]; r.v[3] = src[3];+ r.kind = 0;+ } else {+ const device uchar* rr = ckv + (size_t) sl[jj - n_win] * 288u;+ r.v[0] = *(const device uint4*) (rr + (size_t) pt * 16u);+ r.v[1] = uint4(uint(rr[256u + uint(pt) * 2u]), uint(rr[256u + uint(pt) * 2u + 1u]), 0u, 0u);+ r.kind = 1;+ }+ }++ static inline void ds4_attn_st(thread const Ds4AttnRaw& r, int sr, int pt, threadgroup uint4* tile4) {+ threadgroup uint4* dst = tile4 + (size_t) sr * 64u + (size_t) pt * 4u;+ if (r.kind == 0) {+ dst[0] = r.v[0]; dst[1] = r.v[1]; dst[2] = r.v[2]; dst[3] = r.v[3];+ } else if (r.kind == 1) {+ const float k0 = ds4_e4m3(r.v[1].x) * 16384.0f;+ const float k1 = ds4_e4m3(r.v[1].y) * 16384.0f;+ const uint wv[4] = {r.v[0].x, r.v[0].y, r.v[0].z, r.v[0].w};+ uint o[16];+ for (uint k = 0; k < 16u; ++k) {+ const uint b = wv[k >> 2] >> (8u * (k & 3u));+ const float ks = k < 8u ? k0 : k1;+ const float lo = ds4_e2m1_val14(b) * ks, hi = ds4_e2m1_val14(b >> 4) * ks;+ o[k] = (as_type<uint>(lo) >> 16) | (as_type<uint>(hi) & 0xffff0000u);+ }+ dst[0] = uint4(o[0], o[1], o[2], o[3]);+ dst[1] = uint4(o[4], o[5], o[6], o[7]);+ dst[2] = uint4(o[8], o[9], o[10], o[11]);+ dst[3] = uint4(o[12], o[13], o[14], o[15]);+ } else {+ dst[0] = uint4(0); dst[1] = uint4(0); dst[2] = uint4(0); dst[3] = uint4(0);+ }+ }++ // ---------------------------------------------------------------- sparse attention on the matrix units (G = 8 heads a threadgroup)+ //+ // The same attention as ds4_sparse_attn_g8 (a token, eight heads, the window rows plus the selected compressed rows, softmax with the head's+ // sink) but the dots and the weighted sum run on simdgroup 8x8 matrices: eight rows a pass are staged once as BF16 (exact), q and the+ // softmax weights are split into a BF16 high and low part (so the products keep about 16 bits), accumulation is fp32, and the softmax is the+ // online one (the running maximum rescales the accumulators through a diagonal matrix).++#ifndef DS4_MX_QK_LO++#define DS4_MX_QK_LO 0 // 0: the QK dot uses the BF16 high part of q only (faster, ~8 fewer bits)++#endif++// ATTN_PROBE (function constant 25): a timing probe. 1 skips the P.V accumulate, 2 skips the QK matmul - the numbers++// it produces are wrong on purpose; it exists to say which half of the pass the 20 s of a 32K prompt is in.++constant int ATTN_PROBE [[function_constant(25)]];++constant int attn_probe = is_function_constant_defined(ATTN_PROBE) ? ATTN_PROBE : 0;+++ kernel void ds4_sparse_attn_mx(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],+ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort sg [[simdgroup_index_in_threadgroup]]) {+ threadgroup uint4 tile4[8u * 64u];+ threadgroup float qbuf[4096]; // q hi / lo as BF16 (8 KB each), later the output rows as float+ threadgroup float part[8u * 64u];+ threadgroup float ssm[64];+ threadgroup bfloat phi[64], plo[64];+ threadgroup float dg[64];+ threadgroup float m_s[8], den_s[8], sc_s[8];+ threadgroup bfloat* qhi = (threadgroup bfloat*) qbuf;+ threadgroup bfloat* qlo = qhi + 4096;+ threadgroup const bfloat* kt = (threadgroup const bfloat*) tile4;++ const int hb = int(tgp.x) * 8, t = int(tgp.y);+ int n_win, n_sel, w0;+ ds4_attn_counts(ar, t, n_win, n_sel, w0);+ const device ushort* wbase = win + (size_t) w0 * 512u;+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;+ for (uint x = 0; x < 16u; ++x) {+ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;+ const float v = q[((size_t) t * heads + uint(hb) + h) * 512u + d];+ const bfloat hi = bfloat(v);+ qhi[idx] = hi;- qlo[idx] = bfloat(v - float(hi));++ if (DS4_MX_QK_LO) qlo[idx] = bfloat(v - float(hi)); // (unused when the QK dot takes the high part only)+ }+ if (tid < 8u) {+ m_s[tid] = -1e30f;+ den_s[tid] = 0.0f;+ }+ simdgroup_float8x8 O[8];+ for (int c = 0; c < 8; ++c) O[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ const int sr = int(tid) >> 4, pt = int(tid) & 15;+ const int total = n_win + n_sel;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ Ds4AttnRaw raw;+ raw.kind = 2;+ if (sr < 8) ds4_attn_ld(wbase, ckv, sl, n_win, 0, min(8, total), sr, pt, raw);+ for (int t0 = 0; t0 < total; t0 += 8) {+ const int n = min(8, total - t0);- if (sr < 8) ds4_attn_st(raw, sr, pt, tile4);++ if (sr < 8 && attn_probe != 5) ds4_attn_st(raw, sr, pt, tile4);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (sr < 8) ds4_attn_ld(wbase, ckv, sl, n_win, t0 + 8, min(8, total - t0 - 8), sr, pt, raw); // the next pass's rows, in flight during this one- { // S = q . K^T over this simdgroup's eighth of the 512 values++ if (attn_probe != 2 && attn_probe != 6) { // S = q . K^T over this simdgroup's eighth of the 512 values+ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);+ for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {+ simdgroup_bfloat8x8 a_hi, a_lo, b;+ simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);+ simdgroup_load(a_hi, qhi, 512, ulong2(ulong(kb) * 8ul, 0ul));+ simdgroup_load(a_lo, qlo, 512, ulong2(ulong(kb) * 8ul, 0ul));+ simdgroup_multiply_accumulate(acc, a_hi, b, acc);- simdgroup_multiply_accumulate(acc, a_lo, b, acc);++ if (DS4_MX_QK_LO) simdgroup_multiply_accumulate(acc, a_lo, b, acc); // (the low part of q: exactness vs speed)+ }+ simdgroup_store(acc, part + uint(sg) * 64u, 8);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- if (tid < 64u) {++ if (tid < 64u && attn_probe != 4 && attn_probe != 6) {+ float s = 0.0f;+ for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];+ ssm[tid] = s * scale;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);- { // head sg's online softmax over the pass's rows++ if (attn_probe != 3) { // head sg's online softmax over the pass's rows+ const bool valid = int(lane) < n;+ const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;+ const float m = ds4_attn_warp_max(s);+ const float m_old = m_s[sg];+ const float m_new = fmax(m_old, m);+ const float p = valid ? exp(s - m_new) : 0.0f;+ const float ps = ds4_warp_sum(p);+ if (lane < 8u) {+ const bfloat hi = bfloat(p);+ phi[uint(sg) * 8u + uint(lane)] = hi;+ plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));+ }+ if (lane == 0u) {+ const float cs = exp(m_old - m_new);+ sc_s[sg] = cs;+ den_s[sg] = den_s[sg] * cs + ps;+ m_s[sg] = m_new;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[tid >> 3] : 0.0f;+ threadgroup_barrier(mem_flags::mem_threadgroup);- { // O = diag(rescale) O + P V on this simdgroup's 64 columns++ if (attn_probe != 1 && attn_probe != 6) { // O = diag(rescale) O + P V on this simdgroup's 64 columns+ simdgroup_float8x8 D;+ simdgroup_bfloat8x8 ph, pl;+ simdgroup_load(D, dg, 8);+ simdgroup_load(ph, phi, 8);+ simdgroup_load(pl, plo, 8);+ for (int c = 0; c < 8; ++c) {+ simdgroup_float8x8 tmp;+ simdgroup_multiply(tmp, D, O[c]);+ simdgroup_bfloat8x8 v;+ simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));+ simdgroup_multiply_accumulate(tmp, ph, v, tmp);+ #if DS4_MX_PLO+ simdgroup_multiply_accumulate(tmp, pl, v, tmp);+ #endif+ O[c] = tmp;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ for (int c = 0; c < 8; ++c) simdgroup_store(O[c], qbuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint x = 0; x < 16u; ++x) {+ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;+ const float den = den_s[h] + exp(sink[uint(hb) + h] - m_s[h]);+ out[((size_t) t * heads + uint(hb) + h) * 512u + d] = qbuf[idx] / den;+ }+ }+++// TWO QUERIES A THREADGROUP, written out in full. The staged keys are shared; each query keeps its own accumulators,++// its own online-softmax state and its own key count (the causal shape makes the second query's set a superset, so the++// union staged is the second query's own set and the first masks its tail).++kernel void ds4_sparse_attn_mx2(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],++ device const uchar* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],++ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],++ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],++ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],++ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],++ ushort lane [[thread_index_in_simdgroup]], ushort sg [[simdgroup_index_in_threadgroup]]) {++ threadgroup uint4 tile4[8u * 64u];++ threadgroup float qbuf[4096]; // two queries' q as BF16 (8 KB each), later a query's output rows++ threadgroup float part[8u * 64u];++ threadgroup float ssm[64];++ threadgroup bfloat phi[64], plo[64];++ threadgroup float dg[64];++ threadgroup float m_s[16], den_s[16], sc_s[16];++ threadgroup bfloat* qhiA = (threadgroup bfloat*) qbuf;++ threadgroup bfloat* qhiB = qhiA + 4096; // a query's q is 4096 bf16++ threadgroup float* obuf = qbuf; // 8 heads x 512 floats: qbuf again, once the key loop is done++ threadgroup const bfloat* kt = (threadgroup const bfloat*) tile4;++++ const int hb = int(tgp.x) * 8;++ const int tA = int(tgp.y) * 2;++ const int rows_total = (ar.gen == 0 && ar.t0 > 0) ? ar.t0 : 0x7fffffff;++ const int nq = (tA + 1 < rows_total) ? 2 : 1;++ int nwA, nsA, w0A, nwB, nsB, w0B;++ ds4_attn_counts(ar, tA, nwA, nsA, w0A);++ nwB = nwA; nsB = nsA; w0B = w0A;++ if (nq > 1) ds4_attn_counts(ar, tA + 1, nwB, nsB, w0B);++ const int totalA = nwA + nsA;++ const int totalB = (nq > 1) ? nwB + nsB : totalA;++ const int total = totalA > totalB ? totalA : totalB;++ const device ushort* wbase = win + (size_t) w0A * 512u;++ const device int* sl = sel + (size_t) tA * (size_t) sel_stride;++ for (uint x = 0; x < 16u; ++x) {++ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;++ qhiA[idx] = bfloat(q[((size_t) tA * heads + uint(hb) + h) * 512u + d]);++ if (nq > 1) qhiB[idx] = bfloat(q[((size_t) (tA + 1) * heads + uint(hb) + h) * 512u + d]);++ }++ if (tid < 16u) {++ m_s[tid] = -1e30f;++ den_s[tid] = 0.0f;++ }++ simdgroup_float8x8 OA[8], OB[8];++ for (int c = 0; c < 8; ++c) {++ OA[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);++ OB[c] = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);++ }++ const int sr = int(tid) >> 4, pt = int(tid) & 15;++ threadgroup_barrier(mem_flags::mem_threadgroup);++ Ds4AttnRaw raw;++ raw.kind = 2;++ if (sr < 8) ds4_attn_ld(wbase, ckv, sl, nwB, 0, min(8, total), sr, pt, raw);++ for (int t0 = 0; t0 < total; t0 += 8) {++ const int np = min(8, total - t0);++ (void) np;++ if (sr < 8) ds4_attn_st(raw, sr, pt, tile4);++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (sr < 8) ds4_attn_ld(wbase, ckv, sl, nwB, t0 + 8, min(8, total - t0 - 8), sr, pt, raw);++ { // query A: S = q . K^T++ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);++ for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {++ simdgroup_bfloat8x8 a_hi, b;++ simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);++ simdgroup_load(a_hi, qhiA, 512, ulong2(ulong(kb) * 8ul, 0ul));++ simdgroup_multiply_accumulate(acc, a_hi, b, acc);++ }++ simdgroup_store(acc, part + uint(sg) * 64u, 8);++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tid < 64u) {++ float s = 0.0f;++ for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];++ ssm[tid] = s * scale;++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ { // query A: online softmax++ const int nq_here = min(8, totalA - t0);++ const bool valid = int(lane) < nq_here;++ const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;++ const float m = ds4_attn_warp_max(s);++ const float m_old = m_s[0 + sg];++ const float m_new = fmax(m_old, m);++ const float p = valid ? exp(s - m_new) : 0.0f;++ const float ps = ds4_warp_sum(p);++ if (lane < 8u) {++ const bfloat hi = bfloat(p);++ phi[uint(sg) * 8u + uint(lane)] = hi;++ plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));++ }++ if (lane == 0u) {++ const float cs = exp(m_old - m_new);++ sc_s[0 + sg] = cs;++ den_s[0 + sg] = den_s[0 + sg] * cs + ps;++ m_s[0 + sg] = m_new;++ }++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[0 + (tid >> 3)] : 0.0f;++ threadgroup_barrier(mem_flags::mem_threadgroup);++ { // query A: O = diag(rescale) O + P V++ simdgroup_float8x8 D;++ simdgroup_bfloat8x8 ph, pl;++ simdgroup_load(D, dg, 8);++ simdgroup_load(ph, phi, 8);++ simdgroup_load(pl, plo, 8);++ for (int c = 0; c < 8; ++c) {++ simdgroup_float8x8 tmp;++ simdgroup_multiply(tmp, D, OA[c]);++ simdgroup_bfloat8x8 v;++ simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));++ simdgroup_multiply_accumulate(tmp, ph, v, tmp);++#if DS4_MX_PLO++ simdgroup_multiply_accumulate(tmp, pl, v, tmp);++#endif++ OA[c] = tmp;++ }++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (nq > 1) {++ { // query B: S = q . K^T++ simdgroup_float8x8 acc = make_filled_simdgroup_matrix<float, 8, 8>(0.0f);++ for (int kb = int(sg) * 8; kb < int(sg) * 8 + 8; ++kb) {++ simdgroup_bfloat8x8 a_hi, b;++ simdgroup_load(b, kt, 512, ulong2(ulong(kb) * 8ul, 0ul), true);++ simdgroup_load(a_hi, qhiB, 512, ulong2(ulong(kb) * 8ul, 0ul));++ simdgroup_multiply_accumulate(acc, a_hi, b, acc);++ }++ simdgroup_store(acc, part + uint(sg) * 64u, 8);++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tid < 64u) {++ float s = 0.0f;++ for (uint g = 0; g < 8u; ++g) s += part[g * 64u + tid];++ ssm[tid] = s * scale;++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ { // query B: online softmax++ const int nq_here = min(8, totalB - t0);++ const bool valid = int(lane) < nq_here;++ const float s = valid ? ssm[uint(sg) * 8u + uint(lane)] : -INFINITY;++ const float m = ds4_attn_warp_max(s);++ const float m_old = m_s[8 + sg];++ const float m_new = fmax(m_old, m);++ const float p = valid ? exp(s - m_new) : 0.0f;++ const float ps = ds4_warp_sum(p);++ if (lane < 8u) {++ const bfloat hi = bfloat(p);++ phi[uint(sg) * 8u + uint(lane)] = hi;++ plo[uint(sg) * 8u + uint(lane)] = bfloat(p - float(hi));++ }++ if (lane == 0u) {++ const float cs = exp(m_old - m_new);++ sc_s[8 + sg] = cs;++ den_s[8 + sg] = den_s[8 + sg] * cs + ps;++ m_s[8 + sg] = m_new;++ }++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tid < 64u) dg[tid] = (tid >> 3) == (tid & 7u) ? sc_s[8 + (tid >> 3)] : 0.0f;++ threadgroup_barrier(mem_flags::mem_threadgroup);++ { // query B: O = diag(rescale) O + P V++ simdgroup_float8x8 D;++ simdgroup_bfloat8x8 ph, pl;++ simdgroup_load(D, dg, 8);++ simdgroup_load(ph, phi, 8);++ simdgroup_load(pl, plo, 8);++ for (int c = 0; c < 8; ++c) {++ simdgroup_float8x8 tmp;++ simdgroup_multiply(tmp, D, OB[c]);++ simdgroup_bfloat8x8 v;++ simdgroup_load(v, kt, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));++ simdgroup_multiply_accumulate(tmp, ph, v, tmp);++#if DS4_MX_PLO++ simdgroup_multiply_accumulate(tmp, pl, v, tmp);++#endif++ OB[c] = tmp;++ }++ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ }++ }++ for (int c = 0; c < 8; ++c) simdgroup_store(OA[c], obuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));++ threadgroup_barrier(mem_flags::mem_threadgroup);++ for (uint x = 0; x < 16u; ++x) {++ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;++ out[((size_t) tA * heads + uint(hb) + h) * 512u + d] = obuf[idx] / (den_s[h] + exp(sink[uint(hb) + h] - m_s[h]));++ }++ if (nq > 1) {++ threadgroup_barrier(mem_flags::mem_threadgroup);++ for (int c = 0; c < 8; ++c) simdgroup_store(OB[c], obuf, 512, ulong2(ulong(sg) * 64ul + ulong(c) * 8ul, 0ul));++ threadgroup_barrier(mem_flags::mem_threadgroup);++ for (uint x = 0; x < 16u; ++x) {++ const uint idx = tid * 16u + x, h = idx >> 9, d = idx & 511u;++ out[((size_t) (tA + 1) * heads + uint(hb) + h) * 512u + d] =++ obuf[idx] / (den_s[8u + h] + exp(sink[uint(hb) + h] - m_s[8u + h]));++ }++ }++}+++ // kContig == false (the row set is BF16 already): one head and query a threadgroup, a warp a row's dot and warp 0 the+ // softmax, every thread two output values.+ static inline void ds4_sparse_bf16_run(device const float* q, device const ushort* win, device const ushort* ckv,+ device const int* sel, uint sel_stride, constant Ds4AttnRows& ar,+ device const float* sink, uint heads, float scale, device float* out,+ threadgroup float* qs, threadgroup float* sc, threadgroup int* rowix,+ threadgroup float* m_run_s, threadgroup float* tscale_s,+ threadgroup float* den_s, uint2 tgp, uint tid, ushort lane_in,+ ushort wp_in) {+ const int h = int(tgp.x), t = int(tgp.y);+ int n_win, n_sel, w0;+ ds4_attn_counts(ar, t, n_win, n_sel, w0);+ const device ushort* wbase = win + (size_t) w0 * 512u;+ const device int* sl = sel + (size_t) t * (size_t) sel_stride;+ const uint lane = uint(lane_in), wp = uint(wp_in);+ for (uint i = tid; i < 512u; i += 256u) qs[i] = q[((size_t) t * heads + uint(h)) * 512u + i];+ if (tid == 0u) {+ m_run_s[0] = -1e30f;+ den_s[0] = 0.0f;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float o0 = 0.0f, o1 = 0.0f;+ const int c = int(tid) * 2;+ const int total = n_win + n_sel;+ for (int t0 = 0; t0 < total; t0 += 64) {+ const int nt = min(64, total - t0);+ if (tid < uint(nt)) {+ const int j = t0 + int(tid);+ rowix[tid] = j < n_win ? -1 - j : sl[j - n_win];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int jj = int(wp); jj < nt; jj += 8) {+ const int code = rowix[jj];+ const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u+ : ckv + (size_t) code * 512u;+ const device uint4* rp = (const device uint4*) (row + lane * 16u);+ const uint4 a = rp[0], b = rp[1];+ const uint av[4] = {a.x, a.y, a.z, a.w}, bv[4] = {b.x, b.y, b.z, b.w};+ float s = 0.0f;+ for (int x = 0; x < 4; ++x) {+ s += qs[lane * 16u + uint(2 * x)] * ds4_bf16_u32(av[x] & 0xffffu);+ s += qs[lane * 16u + uint(2 * x + 1)] * ds4_bf16_u32(av[x] >> 16);+ }+ for (int x = 0; x < 4; ++x) {+ s += qs[lane * 16u + uint(8 + 2 * x)] * ds4_bf16_u32(bv[x] & 0xffffu);+ s += qs[lane * 16u + uint(9 + 2 * x)] * ds4_bf16_u32(bv[x] >> 16);+ }+ s = ds4_warp_sum(s);+ if (lane == 0u) sc[jj] = s * scale;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (wp == 0u) {+ float m = -INFINITY;+ for (int j = int(lane); j < nt; j += 32) m = fmax(m, sc[j]);+ m = ds4_attn_warp_max(m);+ const float m_old = m_run_s[0];+ const float m_new = fmax(m_old, m);+ float s = 0.0f;+ for (int j = int(lane); j < nt; j += 32) {+ const float p = exp(sc[j] - m_new);+ sc[j] = p;+ s += p;+ }+ s = ds4_warp_sum(s);+ if (lane == 0u) {+ tscale_s[0] = exp(m_old - m_new);+ den_s[0] = den_s[0] * tscale_s[0] + s;+ m_run_s[0] = m_new;+ }+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const float cs = tscale_s[0];+ o0 *= cs;+ o1 *= cs;+ for (int j = 0; j < nt; ++j) {+ const int code = rowix[j];+ const device ushort* row = code < 0 ? wbase + (size_t) (-1 - code) * 512u+ : ckv + (size_t) code * 512u;+ const uint pr = *(const device uint*) (row + c);+ o0 = fma(sc[j], ds4_bf16_u32(pr & 0xffffu), o0);+ o1 = fma(sc[j], ds4_bf16_u32(pr >> 16), o1);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ const float den = den_s[0] + exp(sink[h] - m_run_s[0]);+ device float* ov = out + ((size_t) t * heads + uint(h)) * 512u;+ ov[c] = o0 / den;+ ov[c + 1] = o1 / den;+ }++ kernel void ds4_sparse_attn_bf16(device const float* q [[buffer(0)]], device const ushort* win [[buffer(1)]],+ device const ushort* ckv [[buffer(2)]], device const int* sel [[buffer(3)]],+ constant uint& sel_stride [[buffer(4)]], constant Ds4AttnRows& ar [[buffer(5)]],+ device const float* sink [[buffer(6)]], constant uint& heads [[buffer(7)]],+ constant float& scale [[buffer(8)]], device float* out [[buffer(9)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {+ threadgroup float qs[512];+ threadgroup float sc[64];+ threadgroup int rowix[64];+ threadgroup float m_run_s, tscale_s, den_s;+ ds4_sparse_bf16_run(q, win, ckv, sel, sel_stride, ar, sink, heads, scale, out, qs, sc, rowix, &m_run_s,+ &tscale_s, &den_s, tgp, tid, lane, wp);+ }++ // ds4_kernels.cu:461 route_kernel - softplus scores, their top k with bias, weights from the unbiased scores. One+ // threadgroup (512) a token, as the CUDA kernel; rows 1024 values apart, the picks kPicks = 8 apart.+ kernel void ds4_route(device const float* logits [[buffer(0)]], device const float* bias [[buffer(1)]],+ constant uint& n [[buffer(2)]], constant uint& k [[buffer(3)]], constant float& route_scale [[buffer(4)]],+ device int* idx [[buffer(5)]], device float* w [[buffer(6)]], device const float* bias_vl [[buffer(7)]],+ constant uint& has_vl [[buffer(8)]], device const uint* vlw [[buffer(9)]],+ uint tg [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort wp [[simdgroup_index_in_threadgroup]]) {+ threadgroup float sc[1024], bs[1024];+ threadgroup float bv[32];+ threadgroup int bi[32];+ threadgroup int pick[16];+ const uint t = tg;+ device const float* l = logits + (size_t) t * 1024u;+ device const float* bp = bias;+ if (has_vl != 0u && ((vlw[t >> 5u] >> (t & 31u)) & 1u) != 0u) bp = bias_vl;+ for (uint i = tid; i < n; i += 512u) {+ const float v = l[i];+ const float sp = v > 20.0f ? v : log(1.0f + exp(v));+ sc[i] = sqrt(sp);+ bs[i] = sc[i] + bp[i];+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (uint r = 0; r < k; ++r) {+ float best = -INFINITY;+ int bix = 0x7fffffff;+ for (uint i = tid; i < n; i += 512u)+ if (bs[i] > best || (bs[i] == best && int(i) < bix)) {+ best = bs[i];+ bix = int(i);+ }+ for (uint o = 16u; o > 0u; o >>= 1u) {+ const float ov = simd_shuffle_xor(best, o);+ const int oi = simd_shuffle_xor(bix, o);+ if (ov > best || (ov == best && oi < bix)) {+ best = ov;+ bix = oi;+ }+ }+ if (lane == 0u) {+ bv[wp] = best;+ bi[wp] = bix;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if (tid == 0u) {+ float b = -INFINITY;+ int ix = 0x7fffffff;+ for (uint q = 0; q < 16u; ++q)+ if (bv[q] > b || (bv[q] == b && bi[q] < ix)) {+ b = bv[q];+ ix = bi[q];+ }+ pick[r] = ix;+ bs[ix] = -INFINITY;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ if (tid == 0u) {+ float s = 0.0f;+ for (uint r = 0; r < k; ++r) s += sc[pick[r]];+ for (uint r = 0; r < k; ++r) {+ idx[(size_t) t * 8u + r] = pick[r];+ w[(size_t) t * 8u + r] = sc[pick[r]] / (s + 1e-20f) * route_scale;+ }+ }+ }++ // ds4_kernels.cu:1336 gather_kernel and :1342 combine_kernel - a token's gathered expert-output rows and the MoE's sum.+ struct Ds4Gather {+ int src[16 * 8];+ };+ struct Ds4Combine {+ int n[16];+ int row[16][8];+ float w[16][8];+ int cpu[16];+ };++ kernel void ds4_gather(device const float* x [[buffer(0)]], constant Ds4Gather& g [[buffer(1)]],+ constant uint& cols [[buffer(2)]], device float* out [[buffer(3)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint r = tgp.y;+ const uint c = tgp.x * 256u + tid;+ if (c < cols) out[(size_t) r * cols + c] = x[(size_t) g.src[r] * cols + c];+ }++ kernel void ds4_combine(device const float* ed [[buffer(0)]], device const float* cpu [[buffer(1)]],+ device const float* sh [[buffer(2)]], constant Ds4Combine& cm [[buffer(3)]],+ constant uint& dim [[buffer(4)]], constant uint& has_cpu [[buffer(5)]],+ constant uint& has_sh [[buffer(6)]], device float* out [[buffer(7)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + tid, t = tgp.y;+ if (e >= dim) return;+ float a = 0.0f;+ for (int j = 0; j < cm.n[t]; ++j) a = fma(cm.w[t][j], ed[(size_t) cm.row[t][j] * dim + e], a);+ if (has_cpu != 0u && cm.cpu[t] != 0) a += cpu[(size_t) t * dim + e];+ if (has_sh != 0u) a += sh[(size_t) t * dim + e];+ out[(size_t) t * dim + e] = a;+ }++ // ---------------------------------------------------------------- ds4_kernels.cu:1156 markov_kernel / :1265 confidence_kernel+ //+ // The drafter's head (DSpark's Markov chain and its confidence): markov_add is a warp a vocabulary row - the previous+ // token's E row (BF16) staged as floats in the threadgroup, then E . H[v] added to logits[v] (H is F16, the pack's+ // markov_head.head.weight); confidence is a threadgroup a token over x and the same E row against the head's weights,+ // its block sum through the logistic.+ struct Ds4MarkovJob {+ device float* logits; // [V]: += the bias+ device const ushort* E; // [vocab][R] BF16+ device const ushort* H; // [vocab][R] F16+ device const int* tok; // the previous token (one int)+ int V, R;+ };++ struct Ds4ConfJob {+ device const float* x; // [T][dim]+ device const ushort* E; // [vocab][R] BF16+ device const int* tok; // [T]+ device const float* w; // [dim + R]+ device float* out; // [T]+ int dim, R;+ };++ // a BF16 bit pattern as its float (ds4_kernels.cu:28 bf16f: the top half, no rounding)+ static inline float ds4_bf16f(ushort b) { return as_type<float>(uint(b) << 16u); }++ // a warp a vocabulary row: logits[v] += E[*tok] . H[v]+ kernel void ds4_markov_add(constant Ds4MarkovJob& j [[buffer(0)]], uint tg [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ threadgroup float er[1024]; // (R <= 1024: the host wrapper refuses more)+ const device ushort* erow = j.E + (size_t) (*j.tok) * (size_t) j.R;+ for (int i = int(tid); i < j.R; i += 256) er[i] = ds4_bf16f(erow[i]);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int v = int(tg) * 8 + int(tid >> 5);+ if (v >= j.V) return;+ const device ushort* hr = j.H + (size_t) v * (size_t) j.R;+ float a = 0.0f;+ for (int c = int(lane) * 8; c < j.R; c += 256) {+ for (int i = 0; i < 8; ++i) a += float(as_type<half>(hr[c + i])) * er[c + i];+ }+ a = ds4_warp_sum(a);+ if (lane == 0u) j.logits[v] += a;+ }++ // a threadgroup a token: the logistic of the head's sum over x and the token's E row+ kernel void ds4_confidence(constant Ds4ConfJob& j [[buffer(0)]], uint tg [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[32];+ const int t = int(tg);+ const device float* xt = j.x + (size_t) t * (size_t) j.dim;+ const device ushort* er = j.E + (size_t) j.tok[t] * (size_t) j.R;+ float a = 0.0f;+ for (int i = int(tid); i < j.dim + j.R; i += 1024) a += j.w[i] * (i < j.dim ? xt[i] : ds4_bf16f(er[i - j.dim]));+ a = ds4_block_sum_n(a, red, lane, w, 32u);+ if (tid == 0u) j.out[t] = 1.0f / (1.0f + exp(-a));+ }++ // embed by raw device addresses (the runtime's MTLResidencySet makes gpuAddress pointers valid kernel arguments)+ struct Ds4EmbedArgs {+ device ushort* rows;+ device float* h;+ uint dim;+ uint hc;+ };+ kernel void ds4_embed_j(device const Ds4EmbedArgs* a [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ ushort lane [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + uint(lane);+ if (e >= a->dim) return;+ const uint t = tgp.y;+ const uint bits = uint(a->rows[(size_t) t * a->dim + e]) << 16;+ const float v = as_type<float>(bits);+ for (uint s = 0; s < a->hc; ++s) a->h[((size_t) t * a->hc + s) * a->dim + e] = v;+ }++ // (diagnostic) write a constant through a device pointer passed as a value, and one passed in a buffer+ kernel void ds4_test_write(device float* out [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {+ if (gid < n) out[gid] = 3.0f;+ }+ kernel void ds4_test_write_v(device const uint64_t* p [[buffer(0)]], constant uint& n [[buffer(1)]], uint gid [[thread_position_in_grid]]) {+ device float* out = (device float*) (uintptr_t) p[0];+ if (gid < n) out[gid] = 7.0f;+ }++ // (diagnostic) report the arguments the kernel received+ kernel void ds4_args_probe(device float* out [[buffer(0)]], constant uint& dim [[buffer(1)]],+ constant uint& hc [[buffer(2)]], uint gid [[thread_position_in_grid]]) {+ if (gid == 0) { out[0] = float(dim); out[1] = float(hc); out[2] = 1.0f; }+ }++ // hc == 4 (every layer of this model): the warp-lane form of the CUDA kernel - lane j*4+k holds comb[j][k], the row and column+ // sums are xor-shuffles (1, 2 within a row; 4, 8 within a column). One simdgroup a token instead of one lane.+ kernel void ds4_hc_split4(device const float* mixes [[buffer(0)]], device const float* scale [[buffer(1)]],+ device const float* base [[buffer(2)]], device float* pre [[buffer(3)]],+ device float* post [[buffer(4)]], device float* comb [[buffer(5)]],+ constant uint& hc [[buffer(6)]], constant uint& iters [[buffer(7)]],+ constant float& eps [[buffer(8)]], uint b [[threadgroup_position_in_grid]],+ uint lane [[thread_index_in_threadgroup]]) {+ const device float* mx = mixes + (size_t) b * 32u;+ if (lane < 4u) {+ pre[(size_t) b * 8u + lane] = 1.0f / (1.0f + exp(-(mx[lane] * scale[0] + base[lane]))) + eps;+ post[(size_t) b * 8u + lane] = 2.0f / (1.0f + exp(-(mx[4u + lane] * scale[1] + base[4u + lane])));+ }+ const bool on = lane < 16u;+ float c = on ? mx[8u + lane] * scale[2] + base[8u + lane] : -3.402823466e+38f;+ float m = fmax(c, simd_shuffle_xor(c, ushort(1)));+ m = fmax(m, simd_shuffle_xor(m, ushort(2)));+ c = on ? exp(c - m) : 0.0f;+ float rs = c + simd_shuffle_xor(c, ushort(1));+ rs += simd_shuffle_xor(rs, ushort(2));+ c = c / rs + eps;+ c = on ? c : 0.0f;+ float cs = c + simd_shuffle_xor(c, ushort(4));+ cs += simd_shuffle_xor(cs, ushort(8));+ c = c / (cs + eps);+ for (uint it = 1; it < iters; ++it) {+ float r = c + simd_shuffle_xor(c, ushort(1));+ r += simd_shuffle_xor(r, ushort(2));+ c = c / (r + eps);+ float s = c + simd_shuffle_xor(c, ushort(4));+ s += simd_shuffle_xor(s, ushort(8));+ c = c / (s + eps);+ }+ if (on) comb[(size_t) b * 64u + lane] = c;+ }++ // Plain byte copies and fills on the context's queue: cudaMemcpyAsync / cudaMemsetAsync between runtime allocations (unified+ // memory: a "device" and a "host" buffer are both just memory) ride in the same command buffer as the kernels around them,+ // so their order is the program order and a layer needs no queue-to-queue handshake.+ kernel void ds4_copy4(device const uint* src [[buffer(0)]], device uint* dst [[buffer(1)]], constant ulong& n4 [[buffer(2)]],+ uint gid [[thread_position_in_grid]]) {+ if (ulong(gid) < n4) dst[gid] = src[gid];+ }+ kernel void ds4_copy1(device const uchar* src [[buffer(0)]], device uchar* dst [[buffer(1)]], constant ulong& n [[buffer(2)]],+ uint gid [[thread_position_in_grid]]) {+ if (ulong(gid) < n) dst[gid] = src[gid];+ }+ kernel void ds4_fill1(device uchar* dst [[buffer(0)]], constant ulong& n [[buffer(1)]], constant uint& v [[buffer(2)]],+ uint gid [[thread_position_in_grid]]) {+ if (ulong(gid) < n) dst[gid] = uchar(v);+ }++ // ---------------------------------------------------------------------------------------------------------------------+ // The prompt chunk's dense GEMM operands (see ds4_gemm_f16_nt on the host): float rows -> half, a quantized matrix -> half rows,+ // strided half rows packed, and the float result scattered into the engine's strided output.++ kernel void ds4_rows_f16(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]],+ device const int* idx [[buffer(2)]], constant uint& has_idx [[buffer(3)]],+ constant uint& ncols [[buffer(4)]], device half* out [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint r = tgp.y, c = tgp.x * 256u + tid;+ if (c >= ncols) return;+ const ulong row = has_idx != 0u ? ulong(idx[r]) : ulong(r);+ out[(ulong) r * ncols + c] = half(x[row * ldx + c]);+ }++ static inline void ds4_kscale2(const device uchar* q, int j, thread int& d, thread int& m) {+ if (j < 4) {+ d = int(q[j]) & 63;+ m = int(q[j + 4]) & 63;+ } else {+ d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);+ m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);+ }+ }++ // a thread a 32-value sub-block of a Q5_K matrix (n rows of k values, ld bytes a row) -> out[n][k] half+ kernel void ds4_dq_q5k(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {+ const uint nb = k / 256u;+ const ulong total = (ulong) n * nb * 8ul;+ if (ulong(gid) >= total) return;+ const uint ib = gid & 7u;+ const uint blk = (gid >> 3) % nb;+ const uint row = uint(ulong(gid) / (8ul * nb));+ const device uchar* p = W + (ulong) row * ld + (ulong) blk * 176ul;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));+ int sc, mm;+ ds4_kscale2(p + 4, int(ib), sc, mm);+ const device uchar* qh = p + 16;+ const device uchar* qs = p + 48 + 32u * (ib >> 1);+ const uint sh = 4u * (ib & 1u);+ device packed_half4* o = (device packed_half4*) (out + (ulong) row * k + (ulong) blk * 256ul + ib * 32u);+ const float ds = d * float(sc), dm = dmin * float(mm);+ for (uint l = 0; l < 32u; l += 4u) { // four values a step: 4-byte loads, 8-byte stores (the same arithmetic a value)+ const uint4 a = uint4(*(const device packed_uchar4*) (qs + l));+ const uint4 b = uint4(*(const device packed_uchar4*) (qh + l));+ const uint4 q = ((a >> sh) & 15u) | (((b >> ib) & 1u) << 4);+ o[l >> 2] = packed_half4(half4(float4(ds) * float4(q) - float4(dm)));+ }+ }++ // Q8_0: 34-byte blocks (fp16 d, 32 int8)+ kernel void ds4_dq_q8(device const uchar* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {+ const uint ng = k / 32u;+ const ulong total = (ulong) n * ng;+ if (ulong(gid) >= total) return;+ const uint g = gid % ng, row = gid / ng;+ const device uchar* p = W + (ulong) row * ld + (ulong) g * 34ul;+ const float d = float(as_type<half>(*(const device ushort*) p));+ device half* o = out + (ulong) row * k + (ulong) g * 32ul;+ for (uint l = 0; l < 32u; ++l) o[l] = half(d * float(char(p[2 + l])));+ }++ // BF16 values (ld values a row) -> half+ kernel void ds4_dq_bf16(device const ushort* W [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],+ constant uint& n [[buffer(3)]], device half* out [[buffer(4)]], uint gid [[thread_position_in_grid]]) {+ const ulong total = (ulong) n * k;+ if (ulong(gid) >= total) return;+ const ulong r = ulong(gid) / k, c = ulong(gid) - r * k;+ out[gid] = half(as_type<float>(uint(W[r * ld + c]) << 16));+ }++ // half rows with a row stride -> contiguous rows+ kernel void ds4_pack_h16(device const ushort* x [[buffer(0)]], constant ulong& ld [[buffer(1)]], constant uint& k [[buffer(2)]],+ device ushort* out [[buffer(3)]], uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint c = tgp.x * 256u + tid;+ if (c < k) out[(ulong) tgp.y * k + c] = x[(ulong) tgp.y * ld + c];+ }++ // y[r][c] = alpha * t[r][c] + beta * y[r][c]+ kernel void ds4_axpby_rows(device const float* t [[buffer(0)]], device float* y [[buffer(1)]], constant ulong& ldy [[buffer(2)]],+ constant uint& n [[buffer(3)]], constant float& alpha [[buffer(4)]], constant float& beta [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint c = tgp.x * 256u + tid;+ if (c >= n) return;+ const ulong yi = (ulong) tgp.y * ldy + c;+ y[yi] = alpha * t[(ulong) tgp.y * n + c] + (beta != 0.0f ? beta * y[yi] : 0.0f);+ }++ // ds4_q8_quant for several jobs in one dispatch (a matvec call's activations): grid (groups, max rows, jobs)+ struct Ds4Q8Spec {+ device const float* x;+ ulong ldx;+ uint rows;+ uint k;+ device char* q;+ device float* d;+ };+ kernel void ds4_q8_quant_multi(device const Ds4Q8Spec* specs [[buffer(0)]], uint3 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]]) {+ const device Ds4Q8Spec& s = specs[tgp.z];+ const uint warp = tid / 32u, lane = tid % 32u;+ const uint b = tgp.x * 8u + warp;+ const uint r = tgp.y;+ if (r >= s.rows || (size_t) b * 32u >= (size_t) s.k) return;+ const device float* xr = s.x + (size_t) r * s.ldx;+ const float v = xr[(size_t) b * 32u + lane];+ const float am = simd_max(fabs(v));+ const float sc = am / 127.0f;+ s.q[(size_t) r * s.k + (size_t) b * 32u + lane] = char(am > 0.0f ? rint(v / sc) : 0.0f);+ if (lane == 0u) s.d[(size_t) r * (s.k / 32u) + b] = sc;+ }diff --cc metal/src/engine/msl/ds4_iq.metalindex 000000000,15f60df47..ea17f7271mode 000000,100644..100644--- a/metal/src/engine/msl/ds4_iq.metal+++ b/metal/src/engine/msl/ds4_iq.metal@@@ -1,0 -1,1060 +1,1092 @@@+ #include <metal_stdlib>+ using namespace metal;+ // ds4_iq.cu:809 mv_kernel - the I-quant expert matvec. Same frame as the MXFP4/Q8_0 ones (a warp a row, a lane strides the+ // 256-value blocks, sixteen accumulator columns for the token window), with the block decoder chosen per job by b.type:+ // 16 IQ2_XXS (66 B), 17 IQ2_XS (74 B), 22 IQ2_S (82 B). The activations are the quantiser's int8 (xq) with their scales (xd),+ // so each sub-group's dot is integer and the two scales fold in at the end - the same arithmetic as the float probes, at a+ // quarter of the traffic.+ struct Ds4MvJob {+ device const uchar* w;+ ulong ld;+ int type, k, n, ncol;+ device const char* xq;+ device const float* xd;+ device float* y;+ ulong ycs;+ device const float* bias;+ float alpha;+ };++ kernel void ds4_iq_mv(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nsb = uint(b.k) / 32u;+ const uint nblk = uint(b.k) / 256u;+ const uint bs = b.type == 16 ? 66u : (b.type == 17 ? 74u : (b.type == 18 ? 98u : 82u));+ const device uchar* w = b.w + (size_t) row * b.ld;+ float acc[16];+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;+ for (uint blk = lane; blk < nblk; blk += 32u) {+ const device uchar* p = w + (size_t) blk * bs;+ const float d = float(as_type<half>(*(const device ushort*) p));+ for (uint ib = 0; ib < 8u; ++ib) {+ if (b.type == 18) { // IQ3_XXS: d, qs[64] (two grid bytes a 8 values), scales_and_signs[8] words (top nibble the scale)+ const device uchar* qs = p + 2;+ const uint w0 = uint(qs[8 * ib]) | (uint(qs[8 * ib + 1]) << 8) | (uint(qs[8 * ib + 2]) << 16) | (uint(qs[8 * ib + 3]) << 24);+ const uint w1 = uint(qs[8 * ib + 4]) | (uint(qs[8 * ib + 5]) << 8) | (uint(qs[8 * ib + 6]) << 16) | (uint(qs[8 * ib + 7]) << 24);+ const uint aux = uint(qs[64 + 4 * ib]) | (uint(qs[65 + 4 * ib]) << 8) | (uint(qs[66 + 4 * ib]) << 16) | (uint(qs[67 + 4 * ib]) << 24);+ const float db3 = d * (0.5f + float(aux >> 28)) * 0.5f;+ for (uint l = 0; l < 4u; ++l) {+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];+ const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + off;+ int sum = 0;+ for (uint j = 0; j < 4u; ++j) {+ int v0 = int((g0 >> (8u * j)) & 0xffu), v1 = int((g1 >> (8u * j)) & 0xffu);+ if ((sgn & (1u << j)) != 0u) v0 = -v0;+ if ((sgn & (1u << (j + 4u))) != 0u) v1 = -v1;+ sum += v0 * int(xq[j]) + v1 * int(xq[j + 4u]);+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * db3, float(sum), acc[c]);+ }+ }+ continue;+ }+ float db0, db1;+ uint idx[4];+ uint sg[4];+ if (b.type == 16) {+ const uint aux0 = uint(p[2 + 8 * ib]) | (uint(p[3 + 8 * ib]) << 8) | (uint(p[4 + 8 * ib]) << 16) |+ (uint(p[5 + 8 * ib]) << 24); // (8 bytes a sub-block: grid bytes, then signs and scale)+ const uint aux1 = uint(p[6 + 8 * ib]) | (uint(p[7 + 8 * ib]) << 8) | (uint(p[8 + 8 * ib]) << 16) |+ (uint(p[9 + 8 * ib]) << 24);+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;+ db0 = db1 = db;+ for (uint l = 0; l < 4u; ++l) {+ idx[l] = (aux0 >> (8u * l)) & 0xffu;+ sg[l] = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);+ }+ } else if (b.type == 17) {+ const device ushort* qs = (const device ushort*) (p + 2);+ const device uchar* sc = p + 66;+ db0 = d * (0.5f + float(sc[ib] & 0xfu)) * 0.25f;+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const uint q = uint(qs[4u * ib + l]);+ idx[l] = q & 511u;+ sg[l] = uint(ksigns_iq2xs[(q >> 9) & 127u]); // the sign byte of the 7 stored bits (bit j set: value j negated)+ }+ } else {+ const device uchar* qs = p + 2;+ const device uchar* sgb = qs + 32;+ const device uchar* qh = p + 66;+ const device uchar* sc = p + 74;+ db0 = d * (0.5f + float(sc[ib] & 0xfu)) * 0.25f;+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ idx[l] = uint(qs[4u * ib + l]) | ((uint(qh[ib]) << (8u - 2u * l)) & 0x300u);+ sg[l] = uint(sgb[4u * ib + l]);+ }+ }+ for (uint l = 0; l < 4u; ++l) {+ ulong g;+ if (b.type == 16) g = iq2xxs_grid[idx[l]];+ else if (b.type == 17) g = iq2xs_grid[idx[l]];+ else g = iq2s_grid[idx[l] & 1023u];+ const float db = (l < 2u) ? db0 : db1;+ const uint off = (size_t) blk * 256u + ib * 32u + l * 8u;+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + off;+ int sum = 0;+ for (uint j = 0; j < 8u; ++j) {+ const int code = int((g >> (8u * j)) & 0xffu);+ const bool neg = b.type == 16 ? ((sg[l] & uint(kmask_iq2xs[j])) != 0u)+ : ((sg[l] & (1u << j)) != 0u);+ sum += (neg ? -code : code) * int(xq[j]);+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * db, float(sum), acc[c]);+ }+ }+ }+ }+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+ }++ // Q5_K / Q6_K (the dense matrices' K-quants; ggml blocks of 176 / 210 bytes): the same warp a row frame, a lane a+ // 256-value block, each value's code times its activation's int8. Q5_K's weight is d*sc*q - dmin*m (both parts of the+ // 32-value group's scales), Q6_K's d*sc*(q - 32).+ static inline void ds4_k_scale(const device uchar* q, int j, thread int& d, thread int& m) {+ if (j < 4) {+ d = int(q[j]) & 63;+ m = int(q[j + 4]) & 63;+ } else {+ d = (int(q[j + 4]) & 0xf) | ((int(q[j - 4]) >> 6) << 4);+ m = (int(q[j + 4]) >> 4) | ((int(q[j]) >> 6) << 4);+ }+ }++ kernel void ds4_iq_mv_k(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nsb = uint(b.k) / 32u;+ const uint nblk = uint(b.k) / 256u;+ const uint bs = b.type == 13 ? 176u : 210u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ float acc[16];+ for (uint c = 0; c < 16u; ++c) acc[c] = 0.0f;+ for (uint blk = lane; blk < nblk; blk += 32u) {+ const device uchar* p = w + (size_t) blk * bs;+ if (b.type == 13) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));+ const device uchar* scales = p + 4;+ const device uchar* qh = p + 16;+ const device uchar* qs = p + 48;+ for (uint ib = 0; ib < 8u; ++ib) {+ int sc, mm;+ ds4_k_scale(scales, int(ib), sc, mm);+ const device uchar* qsp = qs + 32u * (ib >> 1);+ const uint sh = 4u * (ib & 1u);+ for (uint t = 0; t < 8u; ++t) {+ const uint q4 = (*(const device uint*) (qsp + 4u * t) >> sh) & 0x0f0f0f0fu;+ const uint h4 = (*(const device uint*) (qh + 4u * t) >> ib) & 0x01010101u;+ const uint q = q4 | (h4 << 4);+ const uint off = blk * 256u + ib * 32u + t * 4u;+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + off;+ int sumq = 0, sumx = 0;+ for (uint j = 0; j < 4u; ++j) {+ const int code = int((q >> (8u * j)) & 0xffu);+ const int xv = int(xq[j]);+ sumq += code * xv;+ sumx += xv;+ }+ const float dot = d * float(sc) * float(sumq) - dmin * float(mm) * float(sumx);+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)], dot, acc[c]);+ }+ }+ }+ } else {+ const float d = float(as_type<half>(*(const device ushort*) (p + 208)));+ const device uchar* ql = p;+ const device uchar* qh = p + 128;+ const device uchar* scl = p + 192;+ for (uint ib = 0; ib < 8u; ++ib) {+ const uint h = ib >> 2, quarter = ib & 3u;+ const device uchar* qlp = ql + 64u * h + ((quarter & 1u) ? 32u : 0u);+ const device uchar* qhp = qh + 32u * h;+ const uint sh = 4u * (quarter >> 1);+ const int s0 = int(((const device char*) scl)[8u * h + 2u * quarter]);+ const int s1 = int(((const device char*) scl)[8u * h + 2u * quarter + 1u]);+ for (uint t = 0; t < 8u; ++t) {+ const uint x = ((*(const device uint*) (qlp + 4u * t) >> sh) & 0x0f0f0f0fu) |+ (((*(const device uint*) (qhp + 4u * t) >> (2u * quarter)) & 0x03030303u) << 4);+ const uint off = blk * 256u + ib * 32u + t * 4u;+ const float s = float(t < 4u ? s0 : s1);+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ const device char* xq = b.xq + (size_t) c * b.k + off;+ int sumq = 0;+ for (uint j = 0; j < 4u; ++j) {+ const int code = int((x >> (8u * j)) & 0xffu) - 32;+ sumq += code * int(xq[j]);+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + (off / 32u)] * d * s, float(sumq), acc[c]);+ }+ }+ }+ }+ }+ for (uint c = 0; c < 16u; ++c) {+ if (int(c) >= b.ncol) break;+ float v = acc[c];+ v += simd_shuffle_xor(v, 16);+ v += simd_shuffle_xor(v, 8);+ v += simd_shuffle_xor(v, 4);+ v += simd_shuffle_xor(v, 2);+ v += simd_shuffle_xor(v, 1);+ if (lane == 0u) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+ }++ // ---------------------------------------------------------------- ds4_iq.cu:536 gate_up_kernel / :588 down_kernel+ //+ // The prompt / window path's expert GEMM (ds4_iq.cu's launch_gu / launch_dn): a threadgroup of 256 = 8 warps, a warp+ // RW rows of the gate and of the up (down: RW rows of the down), a lane 32 / RW sub-blocks of the row, and the group's+ // entries NE at a pass so that a sub-block the weights decode once serves them all. Grid (rows / (8 RW), groups).+ //+ // The group's blob and its entries are the CUDA kernel's `group` (ds4_iq.cu:512): a plan group (grp_ptr null) is one+ // expert whose entry is the group's own index and whose activation row is 0, a window group's entries come from+ // grp_ent[grp_start[g] .. grp_start[g + 1]) and an entry e's activation row is e >> 3.+ struct Ds4IqJob {+ device const ulong* plan_ptr; // plan_ptr[g]: the expert's blob (0: the group is skipped)+ device const ulong* grp_ptr; // grp_ptr[g]: a window set's blob+ device const int* grp_start; // a set's first entry (n_groups + 1 of them)+ device const int* grp_ent; // the entries (token * 8 + pick)+ device const int* n_groups;+ device const char* xq; // the int8 activations of a row (n_embd an entry)+ device const float* xd; // their scales (n_embd / 32 a row)+ device const char* hq; // down's int8 input (n_ff an entry)+ device const float* hd;+ device float* h; // [entries][n_ff]: the clamped swiglu+ device float* part; // [entries][n_embd]: each entry's down output+ long tr0, tr1, tr2; // the roles' offsets in a blob+ int n_embd, n_ff;+ int type; // 16 IQ2_XXS, 17 IQ2_XS, 22 IQ2_S (ds4_iq.cuh's kIQ2_*)+ float limit; // swiglu_limit (0: none)+ };++ // a 32-value sub-block as 32 int8 codes (8 packed ints, values 4 w .. 4 w + 3 in word w) and the two halves' scales+ // (ggml's d (2 s + 1) / 8, the .cu's Sub.d * Sub.s0 in one F32: the codes of the second half are scaled by s1)+ struct Ds4IqSub {+ int q[8];+ float s0, s1;+ };++ // the block bytes of the types these kernels take: 16 IQ2_XXS, 17 IQ2_XS, 18 IQ3_XXS, 22 IQ2_S (ds4_iq.cu's Ty<T>::B)+ static inline uint ds4_iq_bs(int type) {+ return type == 16 ? 66u : type == 17 ? 74u : type == 18 ? 98u : 82u;+ }++ // ds4_iq.cu:139 decode<T>: the block's 8 sub-blocks, kb's is the (kb & 7)'th; the grids are ggml-common.h's+ static inline Ds4IqSub ds4_iq_sub(device const uchar* row, uint bs, int kb, int type) {+ Ds4IqSub s;+ const int ib = kb & 7;+ const device uchar* p = row + (size_t) (kb >> 3) * bs;+ const float d = float(as_type<half>(*(const device ushort*) p));+ uint idx[4], sm[4];+ float db0, db1;+ if (type == 18) { // IQ3_XXS: d, qs[64] grid bytes (4 a sub-block), scales[8] (the top nibble the group's s)+ const device uchar* qs = p + 2;+ const uint w0 = uint(qs[8 * ib]) | (uint(qs[8 * ib + 1]) << 8) | (uint(qs[8 * ib + 2]) << 16) |+ (uint(qs[8 * ib + 3]) << 24);+ const uint w1 = uint(qs[8 * ib + 4]) | (uint(qs[8 * ib + 5]) << 8) | (uint(qs[8 * ib + 6]) << 16) |+ (uint(qs[8 * ib + 7]) << 24);+ const uint aux = uint(qs[64 + 4 * ib]) | (uint(qs[65 + 4 * ib]) << 8) | (uint(qs[66 + 4 * ib]) << 16) |+ (uint(qs[67 + 4 * ib]) << 24);+ for (uint l = 0; l < 4u; ++l) {+ const uint w = l < 2u ? w0 : w1, sh = 16u * (l & 1u);+ const uint sg = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);+ const uint g0 = iq3xxs_grid[(w >> sh) & 255u], g1 = iq3xxs_grid[(w >> (sh + 8u)) & 255u];+ int a0 = 0, a1 = 0;+ for (uint j = 0; j < 4u; ++j) {+ int v0 = int((g0 >> (8u * j)) & 0xffu), v1 = int((g1 >> (8u * j)) & 0xffu);+ if ((sg & (1u << j)) != 0u) v0 = -v0;+ if ((sg & (1u << (j + 4u))) != 0u) v1 = -v1;+ a0 |= (v0 & 0xff) << int(8u * j);+ a1 |= (v1 & 0xff) << int(8u * j);+ }+ s.q[2u * l] = a0;+ s.q[2u * l + 1u] = a1;+ }+ const float ds = d * (0.5f + float(aux >> 28)) * 0.5f; // ggml's db: d (0.5 + s) 0.5+ s.s0 = ds;+ s.s1 = ds;+ return s;+ }+ if (type == 16) { // d, qs[8] i16 (4 grid bytes, then 4 sign/scale bytes), scales in the top nibbles+ const uint aux0 = uint(p[2 + 8 * ib]) | (uint(p[3 + 8 * ib]) << 8) | (uint(p[4 + 8 * ib]) << 16) |+ (uint(p[5 + 8 * ib]) << 24);+ const uint aux1 = uint(p[6 + 8 * ib]) | (uint(p[7 + 8 * ib]) << 8) | (uint(p[8 + 8 * ib]) << 16) |+ (uint(p[9 + 8 * ib]) << 24);+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;+ db0 = db1 = db;+ for (uint l = 0; l < 4u; ++l) {+ idx[l] = (aux0 >> (8u * l)) & 0xffu;+ sm[l] = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);+ }+ } else if (type == 17) { // d, qs[16] i16 (9-bit grid, 7 sign bits), scales[8] (two 4-bit)+ const device ushort* qs = (const device ushort*) (p + 2);+ const device uchar* sc = p + 66;+ db0 = d * (0.5f + float(sc[ib] & 0x0fu)) * 0.25f;+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const uint q = uint(qs[4u * uint(ib) + l]);+ idx[l] = q & 511u;+ sm[l] = uint(ksigns_iq2xs[(q >> 9) & 127u]);+ }+ } else { // IQ2_S: d, qs[32] (8-bit grid), signs[32], qh[8] (2 bits more), scales[8]+ const device uchar* qs = p + 2;+ const device uchar* sgb = qs + 32;+ const device uchar* qh = p + 66;+ const device uchar* sc = p + 74;+ db0 = d * (0.5f + float(sc[ib] & 0x0fu)) * 0.25f;+ db1 = d * (0.5f + float(sc[ib] >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ idx[l] = uint(qs[4u * uint(ib) + l]) | ((uint(qh[ib]) << (8u - 2u * l)) & 0x300u);+ sm[l] = uint(sgb[4u * uint(ib) + l]);+ }+ }+ for (uint l = 0; l < 4u; ++l) {+ const ulong g = type == 16 ? iq2xxs_grid[idx[l]] : (type == 17 ? iq2xs_grid[idx[l]] : iq2s_grid[idx[l] & 1023u]);+ int w0 = 0, w1 = 0;+ for (uint j = 0; j < 4u; ++j) {+ int v0 = int((g >> (8u * j)) & 0xffu), v1 = int((g >> (8u * (j + 4u))) & 0xffu);+ if ((sm[l] & (1u << j)) != 0u) v0 = -v0;+ if ((sm[l] & (1u << (j + 4u))) != 0u) v1 = -v1;+ w0 |= (v0 & 0xff) << int(8u * j);+ w1 |= (v1 & 0xff) << int(8u * j);+ }+ s.q[2u * l] = w0;+ s.q[2u * l + 1u] = w1;+ }+ s.s0 = db0;+ s.s1 = db1;+ return s;+ }++ // 4 int8 codes against the 4 int8 activations packed in x (ds4_iq.cu:350's __dp4a: Metal has no such instruction)+ static inline int ds4_iq_dot4(int q, int x) {+ const char4 a = as_type<char4>(q), b = as_type<char4>(x);+ return int(a.x) * int(b.x) + int(a.y) * int(b.y) + int(a.z) * int(b.z) + int(a.w) * int(b.w);+ }++ // ds4_iq.cu:349 sub_dot: the sub-block against its 32 int8 activations (8 packed ints), the scales in F32+ static inline float ds4_iq_dot(Ds4IqSub s, device const int* xp) {+ const int lo = ds4_iq_dot4(s.q[0], xp[0]) + ds4_iq_dot4(s.q[1], xp[1]) + ds4_iq_dot4(s.q[2], xp[2]) ++ ds4_iq_dot4(s.q[3], xp[3]);+ const int hi = ds4_iq_dot4(s.q[4], xp[4]) + ds4_iq_dot4(s.q[5], xp[5]) + ds4_iq_dot4(s.q[6], xp[6]) ++ ds4_iq_dot4(s.q[7], xp[7]);+ return s.s0 * float(lo) + s.s1 * float(hi);+ }++ // ds4_iq.cu:502 lanes_sum: the row's sum over the 32 / RW lanes that share it+ template <int RW>+ static inline float ds4_iq_lsum(float v) {+ for (int o = 16 / RW; o > 0; o >>= 1) v += simd_shuffle_xor(v, ushort(o));+ return v;+ }++ // ds4_iq.cu:497 swiglu: the limit's clamps, then g / (1 + e^-g) * u+ static inline float ds4_iq_swiglu(float gv, float uv, float limit) {+ if (limit > 0.0f) {+ gv = min(gv, limit);+ uv = min(max(uv, -limit), limit);+ }+ return (gv / (1.0f + exp(-gv))) * uv;+ }++ // the group's blob and its entry range (ds4_iq.cu:512 group); false: the threadgroup has nothing to do+ static inline bool ds4_iq_group(constant Ds4IqJob& j, uint g, thread ulong& blob, thread int& m0, thread int& m1) {+ if (j.grp_ptr == nullptr) {+ const ulong p = j.plan_ptr[g];+ if (p == 0ul) return false;+ blob = p;+ m0 = int(g);+ m1 = m0 + 1;+ return true;+ }+ if (int(g) >= *j.n_groups) return false;+ blob = j.grp_ptr[g];+ m0 = j.grp_start[g];+ m1 = j.grp_start[g + 1];+ return true;+ }++ // gate_up_kernel<T, NE> (kGuRW = 1: a warp a row, a lane a sub-block): h = the clamped swiglu of the gate and the up+ kernel void ds4_iq_gu(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ ulong blob;+ int m0, m1;+ if (!ds4_iq_group(j, tgp.y, blob, m0, m1)) return;+ const int row = int(tgp.x) * 8 + int(tid >> 5);+ if (row >= j.n_ff) return;+ const uint bs = ds4_iq_bs(j.type);+ const uint nsb = uint(j.n_embd) / 32u;+ const size_t rb = (size_t) (j.n_embd / 256) * (size_t) bs;+ const device uchar* g = (const device uchar*) blob + (size_t) j.tr0 + (size_t) row * rb;+ const device uchar* u = (const device uchar*) blob + (size_t) j.tr1 + (size_t) row * rb;+ for (int mm = m0; mm < m1; mm += 4) {+ const int ne = min(4, m1 - mm);+ int es[4], xr[4];+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const int e = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];+ es[i] = e;+ xr[i] = j.grp_ptr == nullptr ? 0 : (e >> 3);+ }+ }+ float ag[4] = {0.0f, 0.0f, 0.0f, 0.0f}, au[4] = {0.0f, 0.0f, 0.0f, 0.0f};+ for (uint kb = uint(lane); kb < nsb; kb += 32u) {+ const Ds4IqSub sg = ds4_iq_sub(g, bs, int(kb), j.type);+ const Ds4IqSub su = ds4_iq_sub(u, bs, int(kb), j.type);+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const device int* xp = (const device int*) (j.xq + (size_t) xr[i] * (size_t) j.n_embd ++ (size_t) kb * 32u);+ const float dx = j.xd[(size_t) xr[i] * nsb + kb];+ ag[i] += dx * ds4_iq_dot(sg, xp);+ au[i] += dx * ds4_iq_dot(su, xp);+ }+ }+ }+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const float gv = ds4_iq_lsum<1>(ag[i]), uv = ds4_iq_lsum<1>(au[i]);+ if (lane == 0u) j.h[(size_t) es[i] * (size_t) j.n_ff + (size_t) row] = ds4_iq_swiglu(gv, uv, j.limit);+ }+ }+ }+ }++ // down_kernel<T, NE> (kDnRW = 2: a warp two rows, 16 lanes a row): part = the down against the int8 h+ kernel void ds4_iq_dn(constant Ds4IqJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ ulong blob;+ int m0, m1;+ if (!ds4_iq_group(j, tgp.y, blob, m0, m1)) return;+ const int row = (int(tgp.x) * 8 + int(tid >> 5)) * 2 + int(lane) / 16;+ if (row >= j.n_embd) return;+ const uint kl = uint(lane) % 16u;+ const uint bs = ds4_iq_bs(j.type);+ const uint nsb = uint(j.n_ff) / 32u;+ const size_t rb = (size_t) (j.n_ff / 256) * (size_t) bs;+ const device uchar* w = (const device uchar*) blob + (size_t) j.tr2 + (size_t) row * rb;+ for (int mm = m0; mm < m1; mm += 4) {+ const int ne = min(4, m1 - mm);+ int es[4];+ for (int i = 0; i < 4; ++i)+ if (i < ne) es[i] = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];+ float ad[4] = {0.0f, 0.0f, 0.0f, 0.0f};+ for (uint kb = kl; kb < nsb; kb += 16u) {+ const Ds4IqSub sd = ds4_iq_sub(w, bs, int(kb), j.type);+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const device int* hp = (const device int*) (j.hq + (size_t) es[i] * (size_t) j.n_ff ++ (size_t) kb * 32u);+ ad[i] += j.hd[(size_t) es[i] * nsb + kb] * ds4_iq_dot(sd, hp);+ }+ }+ }+ for (int i = 0; i < 4; ++i) {+ if (i < ne) {+ const float v = ds4_iq_lsum<2>(ad[i]);+ if (kl == 0u) j.part[(size_t) es[i] * (size_t) j.n_embd + (size_t) row] = v;+ }+ }+ }+ }++ // ---------------------------------------------------------------- ds4_gguf.cu:454 mx_gate_up_kernel / :519 mx_down_kernel+ //+ // The MXFP4 expert GEMM (ds4_gguf.cu's moe_gate_up / moe_down / win_experts): the same frame as ds4_iq_gu / ds4_iq_dn+ // above - a threadgroup of 256 is 8 warps, a warp a row of the gate and of the up (down: a row of the down), a lane+ // strides the 32-value blocks of the row, and the group's entries go sixteen at a pass so that a block the weights decode+ // once serves them all. A block is 17 bytes (kMxBytes): the E8M0 scale byte, then 16 bytes whose low nibble is value j and+ // whose high nibble is value j + 16, each an index into the codebook {0, 1, 2, 3, 4, 6, 8, 12} and its negatives - the same+ // decode as ds4_init.metal's ds4_mx_block / ds4_e8m0 (ds4_gguf.cu:143 / :154), kept as int8 codes here because the+ // activations are the quantiser's int8, so the dot is integer (ds4_gguf.cu:225 mx_dot) and the two scales fold in at the+ // end.+ //+ // The group's blob and its entries are read as ds4_iq_group above reads them (ds4_gguf.cu:429 mx_group): a plan group+ // (grp_ptr null) is one expert whose entry is the group's own index and whose activation row is 0, a window group's+ // entries come from grp_ent[grp_start[g] .. grp_start[g + 1]) and an entry e's activation row is e >> 3.+ struct Ds4MxJob {+ device const ulong* plan_ptr; // plan_ptr[g]: the expert's blob (0: the group is skipped)+ device const ulong* grp_ptr; // grp_ptr[g]: a window set's blob+ device const int* grp_start; // a set's first entry (n_groups + 1 of them)+ device const int* grp_ent; // the entries (token * 8 + pick)+ device const int* n_groups;+ device const char* xq; // gate / up's int8 activations (n_embd an entry)+ device const float* xd; // their scales (n_embd / 32 a row)+ device const char* hq; // down's int8 input (n_ff an entry)+ device const float* hd; // its scales+ device float* h; // [entries][n_ff]: the clamped swiglu+ device float* part; // [entries][n_embd]: each entry's down output+ long tr0, tr1, tr2; // the roles' offsets in a blob+ int n_embd, n_ff;+ float limit; // swiglu_limit (0: none)+ };++ // the codebook of a nibble, in index order (ds4_init.metal's lut_init: {0,1,2,3,4,6,8,12}, then its negatives)+ constant char ds4_mx_code[16] = {0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12};++ // 2^(e - 128) (ds4_init.metal:489 ds4_e8m0, ggml's GGML_E8M0_TO_FP32_HALF)+ static inline float ds4_mx_e8m0(uint e) {+ return as_type<float>(e < 2u ? (0x00200000u << e) : ((e - 1u) << 23));+ }++ // a block's 16 code bytes as 8 int8 words: word j the low nibbles of bytes 4 j .. (values 4 j ..), word 4 + j their high+ // nibbles (values 16 + 4 j ..) - the pairs ds4_gguf.cu:225 mx_dot reads xp[j] and xp[j + 4] with+ static inline void ds4_mx_words(device const uchar* p, thread int* q) {+ for (uint j = 0; j < 4u; ++j) {+ uint lo = 0u, hi = 0u;+ for (uint b = 0; b < 4u; ++b) {+ const uint v = uint(p[1u + 4u * j + b]);+ lo |= uint(uchar(ds4_mx_code[v & 15u])) << (8u * b);+ hi |= uint(uchar(ds4_mx_code[v >> 4u])) << (8u * b);+ }+ q[j] = int(lo);+ q[4u + j] = int(hi);+ }+ }++ // a block's 32 int8 codes against its 32 int8 activations (8 packed words, 4 values a word)+ static inline int ds4_mx_dot(thread const int* q, device const int* xp) {+ int s = 0;+ for (uint t = 0; t < 8u; ++t) s += ds4_iq_dot4(q[t], xp[t]);+ return s;+ }++ // the group's blob and its entry range (ds4_iq_group's MX form, ds4_gguf.cu:429 mx_group); false: nothing to do+ static inline bool ds4_mx_group(constant Ds4MxJob& j, uint g, thread ulong& blob, thread int& m0, thread int& m1) {+ if (j.grp_ptr == nullptr) {+ const ulong p = j.plan_ptr[g];+ if (p == 0ul) return false;+ blob = p;+ m0 = int(g);+ m1 = m0 + 1;+ return true;+ }+ if (int(g) >= *j.n_groups) return false;+ blob = j.grp_ptr[g];+ m0 = j.grp_start[g];+ m1 = j.grp_start[g + 1];+ return true;+ }++ // mx_gate_up_kernel<NE> (a warp a row of the gate and of the up, a lane a block a step): h = the clamped swiglu+ kernel void ds4_mx_gu(constant Ds4MxJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ ulong blob;+ int m0, m1;+ if (!ds4_mx_group(j, tgp.y, blob, m0, m1)) return;+ const int row = int(tgp.x) * 8 + int(tid >> 5);+ if (row >= j.n_ff) return;+ const uint nblk = uint(j.n_embd) / 32u;+ const size_t rb = (size_t) nblk * 17u;+ const device uchar* g = (const device uchar*) blob + (size_t) j.tr0 + (size_t) row * rb;+ const device uchar* u = (const device uchar*) blob + (size_t) j.tr1 + (size_t) row * rb;+ for (int mm = m0; mm < m1; mm += 16) {+ const int ne = min(16, m1 - mm);+ int es[16], xr[16];+ for (int i = 0; i < 16; ++i) {+ if (i < ne) {+ const int e = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];+ es[i] = e;+ xr[i] = j.grp_ptr == nullptr ? 0 : (e >> 3);+ }+ }+ float ag[16], au[16];+ for (int i = 0; i < 16; ++i) {+ ag[i] = 0.0f;+ au[i] = 0.0f;+ }+ for (uint b = uint(lane); b < nblk; b += 32u) {+ const device uchar* pg = g + (size_t) b * 17u;+ const device uchar* pu = u + (size_t) b * 17u;+ int wg[8], wu[8];+ ds4_mx_words(pg, wg);+ ds4_mx_words(pu, wu);+ const float dg = ds4_mx_e8m0(uint(pg[0])), du = ds4_mx_e8m0(uint(pu[0]));+ for (int i = 0; i < 16; ++i) {+ if (i < ne) {+ const device int* xp =+ (const device int*) (j.xq + (size_t) xr[i] * (size_t) j.n_embd + (size_t) b * 32u);+ const float dx = j.xd[(size_t) xr[i] * nblk + b];+ ag[i] = fma(dg * dx, float(ds4_mx_dot(wg, xp)), ag[i]);+ au[i] = fma(du * dx, float(ds4_mx_dot(wu, xp)), au[i]);+ }+ }+ }+ for (int i = 0; i < 16; ++i) {+ if (i < ne) {+ const float gv = ds4_iq_lsum<1>(ag[i]), uv = ds4_iq_lsum<1>(au[i]);+ if (lane == 0u) j.h[(size_t) es[i] * (size_t) j.n_ff + (size_t) row] = ds4_iq_swiglu(gv, uv, j.limit);+ }+ }+ }+ }++ // mx_down_kernel<NE> (a warp a row of the down, a lane a block a step): part = the down against the int8 h+ kernel void ds4_mx_dn(constant Ds4MxJob& j [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ ulong blob;+ int m0, m1;+ if (!ds4_mx_group(j, tgp.y, blob, m0, m1)) return;+ const int row = int(tgp.x) * 8 + int(tid >> 5);+ if (row >= j.n_embd) return;+ const uint nblk = uint(j.n_ff) / 32u;+ const size_t rb = (size_t) nblk * 17u;+ const device uchar* w = (const device uchar*) blob + (size_t) j.tr2 + (size_t) row * rb;+ for (int mm = m0; mm < m1; mm += 16) {+ const int ne = min(16, m1 - mm);+ int es[16];+ for (int i = 0; i < 16; ++i)+ if (i < ne) es[i] = j.grp_ptr == nullptr ? (mm + i) : j.grp_ent[mm + i];+ float ad[16];+ for (int i = 0; i < 16; ++i) ad[i] = 0.0f;+ for (uint b = uint(lane); b < nblk; b += 32u) {+ const device uchar* pw = w + (size_t) b * 17u;+ int ww[8];+ ds4_mx_words(pw, ww);+ const float dd = ds4_mx_e8m0(uint(pw[0]));+ for (int i = 0; i < 16; ++i) {+ if (i < ne) {+ const device int* hp =+ (const device int*) (j.hq + (size_t) es[i] * (size_t) j.n_ff + (size_t) b * 32u);+ ad[i] = fma(dd * j.hd[(size_t) es[i] * nblk + b], float(ds4_mx_dot(ww, hp)), ad[i]);+ }+ }+ }+ for (int i = 0; i < 16; ++i) {+ if (i < ne) {+ const float v = ds4_iq_lsum<1>(ad[i]);+ if (lane == 0u) j.part[(size_t) es[i] * (size_t) j.n_embd + (size_t) row] = v;+ }+ }+ }+ }++ // ---------------------------------------------------------------------------------------------------------------------+ // One-activation matvecs (the decode step): every lane of the row's simdgroup works, on a 32-value sub-block (IQ2_XXS) or+ // a 64-value pair (Q5_K), where the 16-column kernels above give a whole 256-value block to a lane (a k = 1280 row used 5+ // lanes of 32) and read it a byte at a time. Same results up to the order of the float sums.++ kernel void ds4_q5k_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint j = lane & 3u; // this lane's pair of sub-blocks: 2j and 2j+1 (one 32-byte run of qs, both nibbles)+ const uint bsel = lane >> 2; // eight blocks a pass+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 8u) {+ const device uchar* p = w + (size_t) blk * 176u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));+ int sc0, mm0, sc1, mm1;+ ds4_k_scale(p + 4, int(2u * j), sc0, mm0);+ ds4_k_scale(p + 4, int(2u * j + 1u), sc1, mm1);+ const device uint4* qs4 = (const device uint4*) (p + 48u + 32u * j);+ const device uint4* qh4 = (const device uint4*) (p + 16);+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 64u * j);+ const uint4 qa = qs4[0], qb = qs4[1], ha = qh4[0], hb = qh4[1];+ const uint4 xa = xq4[0], xb = xq4[1], xc = xq4[2], xd4 = xq4[3]; // sub-block 2j: xa xb; 2j+1: xc xd4+ const uint qsw[8] = {qa.x, qa.y, qa.z, qa.w, qb.x, qb.y, qb.z, qb.w};+ const uint qhw[8] = {ha.x, ha.y, ha.z, ha.w, hb.x, hb.y, hb.z, hb.w};+ const uint xlo[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ const uint xhi[8] = {xc.x, xc.y, xc.z, xc.w, xd4.x, xd4.y, xd4.z, xd4.w};+ int sq0 = 0, sx0 = 0, sq1 = 0, sx1 = 0;+ for (uint t = 0; t < 8u; ++t) {+ const uint q0 = (qsw[t] & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j)) & 0x01010101u) << 4);+ const uint q1 = ((qsw[t] >> 4) & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j + 1u)) & 0x01010101u) << 4);+ for (uint i = 0; i < 4u; ++i) {+ const int x0 = int(char(xlo[t] >> (8u * i))), x1 = int(char(xhi[t] >> (8u * i)));+ sq0 += int((q0 >> (8u * i)) & 0xffu) * x0;+ sq1 += int((q1 >> (8u * i)) & 0xffu) * x1;+ sx0 += x0;+ sx1 += x1;+ }+ }+ acc = fma(b.xd[blk * 8u + 2u * j], d * float(sc0) * float(sq0) - dmin * float(mm0) * float(sx0), acc);+ acc = fma(b.xd[blk * 8u + 2u * j + 1u], d * float(sc1) * float(sq1) - dmin * float(mm1) * float(sx1), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }++ kernel void ds4_iq2xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u; // this lane's 32-value sub-block+ const uint bsel = lane >> 3; // four blocks a pass+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w + (size_t) blk * 66u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device ushort* us = (const device ushort*) (p + 2u + 8u * ib); // (2-byte aligned: four 16-bit loads)+ const uint aux0 = uint(us[0]) | (uint(us[1]) << 16);+ const uint aux1 = uint(us[2]) | (uint(us[3]) << 16);+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int sum = 0;+ for (uint l = 0; l < 4u; ++l) {+ const ulong g = iq2xxs_grid[(aux0 >> (8u * l)) & 0xffu];+ const uint sg = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);+ for (uint h = 0; h < 2u; ++h) {+ const uint gw = uint(g >> (32u * h));+ const uint x = xw[2u * l + h];+ for (uint i = 0; i < 4u; ++i) {+ const int code = int((gw >> (8u * i)) & 0xffu);+ const int xv = int(char(x >> (8u * i)));+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;+ }+ }+ }+ acc = fma(b.xd[blk * 8u + ib] * db, float(sum), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }++ // The other low-bit expert types, one activation (see ds4_iq2xxs_mv1): a lane a 32-value sub-block, four blocks a pass.++ kernel void ds4_iq2xs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u;+ const uint bsel = lane >> 3;+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w + (size_t) blk * 74u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device ushort* qs = (const device ushort*) (p + 2u + 8u * ib);+ const uint sc = uint(p[66u + ib]);+ const float db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f, db1 = d * (0.5f + float(sc >> 4)) * 0.25f;+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int s0 = 0, s1 = 0;+ for (uint l = 0; l < 4u; ++l) {+ const uint q = uint(qs[l]);+ const ulong g = iq2xs_grid[q & 511u];+ const uint sg = uint(ksigns_iq2xs[(q >> 9) & 127u]);+ int sum = 0;+ for (uint h = 0; h < 2u; ++h) {+ const uint gw = uint(g >> (32u * h));+ const uint x = xw[2u * l + h];+ for (uint i = 0; i < 4u; ++i) {+ const int code = int((gw >> (8u * i)) & 0xffu);+ const int xv = int(char(x >> (8u * i)));+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;+ }+ }+ if (l < 2u) s0 += sum; else s1 += sum;+ }+ acc = fma(b.xd[blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }++ kernel void ds4_iq2s_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u;+ const uint bsel = lane >> 3;+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w + (size_t) blk * 82u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device uchar* qs = p + 2u + 4u * ib;+ const device uchar* sgb = p + 34u + 4u * ib;+ const uint qh = uint(p[66u + ib]);+ const uint sc = uint(p[74u + ib]);+ const float db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f, db1 = d * (0.5f + float(sc >> 4)) * 0.25f;+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int s0 = 0, s1 = 0;+ for (uint l = 0; l < 4u; ++l) {+ const uint idx = uint(qs[l]) | ((qh << (8u - 2u * l)) & 0x300u);+ const ulong g = iq2s_grid[idx];+ const uint sg = uint(sgb[l]);+ int sum = 0;+ for (uint h = 0; h < 2u; ++h) {+ const uint gw = uint(g >> (32u * h));+ const uint x = xw[2u * l + h];+ for (uint i = 0; i < 4u; ++i) {+ const int code = int((gw >> (8u * i)) & 0xffu);+ const int xv = int(char(x >> (8u * i)));+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;+ }+ }+ if (l < 2u) s0 += sum; else s1 += sum;+ }+ acc = fma(b.xd[blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }++ kernel void ds4_iq3xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u;+ const uint bsel = lane >> 3;+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w + (size_t) blk * 98u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device uchar* qs = p + 2u;+ const uint w0 = uint(qs[8u * ib]) | (uint(qs[8u * ib + 1u]) << 8) | (uint(qs[8u * ib + 2u]) << 16) | (uint(qs[8u * ib + 3u]) << 24);+ const uint w1 = uint(qs[8u * ib + 4u]) | (uint(qs[8u * ib + 5u]) << 8) | (uint(qs[8u * ib + 6u]) << 16) | (uint(qs[8u * ib + 7u]) << 24);+ const uint aux = uint(qs[64u + 4u * ib]) | (uint(qs[65u + 4u * ib]) << 8) | (uint(qs[66u + 4u * ib]) << 16) | (uint(qs[67u + 4u * ib]) << 24);+ const float db = d * (0.5f + float(aux >> 28)) * 0.5f;+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int sum = 0;+ for (uint l = 0; l < 4u; ++l) {+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];+ const uint xl = xw[2u * l], xh = xw[2u * l + 1u];+ for (uint i = 0; i < 4u; ++i) {+ int v0 = int((g0 >> (8u * i)) & 0xffu), v1 = int((g1 >> (8u * i)) & 0xffu);+ if ((sgn & (1u << i)) != 0u) v0 = -v0;+ if ((sgn & (1u << (i + 4u))) != 0u) v1 = -v1;+ sum += v0 * int(char(xl >> (8u * i))) + v1 * int(char(xh >> (8u * i)));+ }+ }+ acc = fma(b.xd[blk * 8u + ib] * db, float(sum), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }++ // ---------------------------------------------------------------------------------------------------------------------+ // Matvecs for NC activations at once (1, 2, 4, 8): the prompt windows' and the drafter's verify windows' columns. A lane takes a+ // 32-value sub-block (a 64-value pair for Q5_K) of the row, decodes its weights ONCE and dots them with each of the NC columns'+ // int8 activations, so the weight bytes (the cost) are read once for the window. ncol <= NC (the rest of the accumulators idle).++ template <uint NC>+ static inline void ds4_mvn_store(const device Ds4MvJob& b, uint row, ushort lane, thread float (&acc)[NC]) {+ for (uint c = 0; c < NC; ++c) {+ const float v = simd_sum(acc[c]);+ if (lane == 0u && int(c) < b.ncol) b.y[(size_t) c * b.ycs + row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+ }+ }++ template <uint NC>+ kernel void ds4_q5k_mvn(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u, nsb = uint(b.k) / 32u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint j = lane & 3u;+ const uint bsel = lane >> 2;+ float acc[NC];+ for (uint c = 0; c < NC; ++c) acc[c] = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 8u) {+ const device uchar* p = w + (size_t) blk * 176u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));+ int sc0, mm0, sc1, mm1;+ ds4_k_scale(p + 4, int(2u * j), sc0, mm0);+ ds4_k_scale(p + 4, int(2u * j + 1u), sc1, mm1);+ const device uint4* qs4 = (const device uint4*) (p + 48u + 32u * j);+ const device uint4* qh4 = (const device uint4*) (p + 16);+ const uint4 qa = qs4[0], qb = qs4[1], ha = qh4[0], hb = qh4[1];+ const uint qsw[8] = {qa.x, qa.y, qa.z, qa.w, qb.x, qb.y, qb.z, qb.w};+ const uint qhw[8] = {ha.x, ha.y, ha.z, ha.w, hb.x, hb.y, hb.z, hb.w};+ uint q0w[8], q1w[8];+ for (uint t = 0; t < 8u; ++t) {+ q0w[t] = (qsw[t] & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j)) & 0x01010101u) << 4);+ q1w[t] = ((qsw[t] >> 4) & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j + 1u)) & 0x01010101u) << 4);+ }+ for (uint c = 0; c < NC; ++c) {+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) c * b.k + (size_t) blk * 256u + 64u * j);+ const uint4 xa = xq4[0], xb = xq4[1], xc = xq4[2], xd4 = xq4[3];+ const uint xlo[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ const uint xhi[8] = {xc.x, xc.y, xc.z, xc.w, xd4.x, xd4.y, xd4.z, xd4.w};+ int sq0 = 0, sx0 = 0, sq1 = 0, sx1 = 0;+ for (uint t = 0; t < 8u; ++t) {+ for (uint i = 0; i < 4u; ++i) {+ const int x0 = int(char(xlo[t] >> (8u * i))), x1 = int(char(xhi[t] >> (8u * i)));+ sq0 += int((q0w[t] >> (8u * i)) & 0xffu) * x0;+ sq1 += int((q1w[t] >> (8u * i)) & 0xffu) * x1;+ sx0 += x0;+ sx1 += x1;+ }+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + 2u * j], d * float(sc0) * float(sq0) - dmin * float(mm0) * float(sx0), acc[c]);+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + 2u * j + 1u], d * float(sc1) * float(sq1) - dmin * float(mm1) * float(sx1), acc[c]);+ }+ }+ ds4_mvn_store<NC>(b, row, lane, acc);+ }++ // the grid-coded types: decode the sub-block's 32 weights (signs applied) into w[32] ...+ static inline void ds4_w32_iq2xxs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device ushort* us = (const device ushort*) (p + 2u + 8u * ib);+ const uint aux0 = uint(us[0]) | (uint(us[1]) << 16);+ const uint aux1 = uint(us[2]) | (uint(us[3]) << 16);+ db = d * (0.5f + float(aux1 >> 28)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const ulong g = iq2xxs_grid[(aux0 >> (8u * l)) & 0xffu];+ const uint sg = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);+ for (uint i = 0; i < 8u; ++i) {+ const int code = int((g >> (8u * i)) & 0xffu);+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;+ }+ }+ }+ static inline void ds4_w32_iq2xs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device ushort* qs = (const device ushort*) (p + 2u + 8u * ib);+ const uint sc = uint(p[66u + ib]);+ db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f;+ db1 = d * (0.5f + float(sc >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const uint q = uint(qs[l]);+ const ulong g = iq2xs_grid[q & 511u];+ const uint sg = uint(ksigns_iq2xs[(q >> 9) & 127u]);+ for (uint i = 0; i < 8u; ++i) {+ const int code = int((g >> (8u * i)) & 0xffu);+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;+ }+ }+ }+ static inline void ds4_w32_iq2s(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device uchar* qs = p + 2u + 4u * ib;+ const device uchar* sgb = p + 34u + 4u * ib;+ const uint qh = uint(p[66u + ib]);+ const uint sc = uint(p[74u + ib]);+ db0 = d * (0.5f + float(sc & 0xfu)) * 0.25f;+ db1 = d * (0.5f + float(sc >> 4)) * 0.25f;+ for (uint l = 0; l < 4u; ++l) {+ const uint idx = uint(qs[l]) | ((qh << (8u - 2u * l)) & 0x300u);+ const ulong g = iq2s_grid[idx];+ const uint sg = uint(sgb[l]);+ for (uint i = 0; i < 8u; ++i) {+ const int code = int((g >> (8u * i)) & 0xffu);+ w[l * 8u + i] = ((sg >> i) & 1u) != 0u ? -code : code;+ }+ }+ }+ static inline void ds4_w32_iq3xxs(const device uchar* p, uint ib, thread int (&w)[32], thread float& db) {+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device uchar* qs = p + 2u;+ const uint w0 = uint(qs[8u * ib]) | (uint(qs[8u * ib + 1u]) << 8) | (uint(qs[8u * ib + 2u]) << 16) | (uint(qs[8u * ib + 3u]) << 24);+ const uint w1 = uint(qs[8u * ib + 4u]) | (uint(qs[8u * ib + 5u]) << 8) | (uint(qs[8u * ib + 6u]) << 16) | (uint(qs[8u * ib + 7u]) << 24);+ const uint aux = uint(qs[64u + 4u * ib]) | (uint(qs[65u + 4u * ib]) << 8) | (uint(qs[66u + 4u * ib]) << 16) | (uint(qs[67u + 4u * ib]) << 24);+ db = d * (0.5f + float(aux >> 28)) * 0.5f;+ for (uint l = 0; l < 4u; ++l) {+ const uint wv = l < 2u ? w0 : w1, sh = 16u * (l & 1u);+ const uint sgn = uint(ksigns_iq2xs[(aux >> (7u * l)) & 127u]);+ const uint g0 = iq3xxs_grid[(wv >> sh) & 255u], g1 = iq3xxs_grid[(wv >> (sh + 8u)) & 255u];+ for (uint i = 0; i < 4u; ++i) {+ int v0 = int((g0 >> (8u * i)) & 0xffu), v1 = int((g1 >> (8u * i)) & 0xffu);+ if ((sgn & (1u << i)) != 0u) v0 = -v0;+ if ((sgn & (1u << (i + 4u))) != 0u) v1 = -v1;+ w[l * 8u + i] = v0;+ w[l * 8u + 4u + i] = v1;+ }+ }+ }+++++// IQ1_M (the 2.00 bpw pack's 1.75-bit type): the block has no d field - its F16 scale is the top nibbles of its four++// u16 scales - and each 32-value sub-block splits into two 16-value halves with their own 3-bit scale (ggml's++// dequantize_row_iq1_m: dl1 for the first two 8-value groups, dl2 for the last two, and a +-0.125 delta per group).++// The weights are the grid's int8 codes plus that delta; the integer form here (8*code +- 1, scale / 8) is the same++// number with no rounding difference, and keeps the dot in integers like the other types.++static inline ushort ds4_iq1m_scale(const device uchar* p) {++ const uint s0 = uint(p[48]) | (uint(p[49]) << 8), s1 = uint(p[50]) | (uint(p[51]) << 8);++ const uint s2 = uint(p[52]) | (uint(p[53]) << 8), s3 = uint(p[54]) | (uint(p[55]) << 8);++ return ushort((s0 >> 12) | ((s1 >> 8) & 0x00f0u) | ((s2 >> 4) & 0x0f00u) | (s3 & 0xf000u));++}++static inline void ds4_w32_iq1m(const device uchar* p, uint ib, thread int (&w)[32], thread float& db0, thread float& db1) {++ const float d = float(as_type<half>(ds4_iq1m_scale(p)));++ const ushort sc = ((const device ushort*) (p + 48))[ib >> 1];++ db0 = d * float(2u * ((sc >> (6u * (ib & 1u))) & 7u) + 1u) * 0.125f;++ db1 = d * float(2u * ((sc >> (6u * (ib & 1u) + 3u)) & 7u) + 1u) * 0.125f;++ const uint qh0 = p[32u + 2u * ib], qh1 = p[33u + 2u * ib];++ for (uint l = 0; l < 4u; ++l) {++ const uint qhl = l < 2u ? qh0 : qh1;++ const uint idx = uint(p[4u * ib + l]) | ((l & 1u) ? ((qhl << 4u) & 0x700u) : ((qhl << 8u) & 0x700u));++ const bool neg = (l & 1u) ? ((qhl & 0x80u) != 0u) : ((qhl & 0x08u) != 0u);++ const ulong g = iq1s_grid[idx];++ for (uint j = 0; j < 8u; ++j) {++ const uint code = uint((g >> (8u * j)) & 0xffu);++ const int c8 = (code & 0x80u) != 0u ? int(code) - 256 : int(code); // -1 / 0 / +1++ w[l * 8u + j] = 8 * c8 + (neg ? -1 : 1);++ }++ }++}+++ // ... and dot them with each column (the sub-block's four 8-value groups: l < 2 use db0, l >= 2 db1)+ template <uint NC, uint TYPE>+ kernel void ds4_iqg_mvn(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u, nsb = uint(b.k) / 32u;- constexpr uint bs = TYPE == 16 ? 66u : (TYPE == 17 ? 74u : (TYPE == 18 ? 98u : 82u));++ constexpr uint bs = TYPE == 16 ? 66u : (TYPE == 17 ? 74u : (TYPE == 18 ? 98u : (TYPE == 29 ? 56u : 82u)));+ const device uchar* w0p = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u;+ const uint bsel = lane >> 3;+ float acc[NC];+ for (uint c = 0; c < NC; ++c) acc[c] = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w0p + (size_t) blk * bs;+ int w[32];+ float db0, db1;+ if (TYPE == 16) { ds4_w32_iq2xxs(p, ib, w, db0); db1 = db0; }+ else if (TYPE == 17) ds4_w32_iq2xs(p, ib, w, db0, db1);+ else if (TYPE == 22) ds4_w32_iq2s(p, ib, w, db0, db1);++ else if (TYPE == 29) ds4_w32_iq1m(p, ib, w, db0, db1);+ else { ds4_w32_iq3xxs(p, ib, w, db0); db1 = db0; }+ for (uint c = 0; c < NC; ++c) {+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) c * b.k + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int s0 = 0, s1 = 0;+ for (uint t = 0; t < 8u; ++t) {+ int part = 0;+ for (uint i = 0; i < 4u; ++i) part += w[t * 4u + i] * int(char(xw[t] >> (8u * i)));+ if (t < 4u) s0 += part; else s1 += part;+ }+ acc[c] = fma(b.xd[(size_t) c * nsb + blk * 8u + ib], db0 * float(s0) + db1 * float(s1), acc[c]);+ }+ }+ ds4_mvn_store<NC>(b, row, lane, acc);+ }++ #define DS4_MVN_INST(NC) \+ template [[host_name("ds4_q5k_mvn" #NC)]] kernel void ds4_q5k_mvn<NC>(device const uchar*, uint2, uint, ushort); \+ template [[host_name("ds4_iq2xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 16>(device const uchar*, uint2, uint, ushort); \+ template [[host_name("ds4_iq2xs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 17>(device const uchar*, uint2, uint, ushort); \+ template [[host_name("ds4_iq2s_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 22>(device const uchar*, uint2, uint, ushort); \- template [[host_name("ds4_iq3xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 18>(device const uchar*, uint2, uint, ushort);++ template [[host_name("ds4_iq3xxs_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 18>(device const uchar*, uint2, uint, ushort); \++ template [[host_name("ds4_iq1m_mvn" #NC)]] kernel void ds4_iqg_mvn<NC, 29>(device const uchar*, uint2, uint, ushort);+ DS4_MVN_INST(1)+ DS4_MVN_INST(2)+ DS4_MVN_INST(4)+ DS4_MVN_INST(8)diff --cc metal/src/engine/msl/ds4_mm.metalindex 000000000,61ebb303d..dfa7d11ecmode 000000,100644..100644--- a/metal/src/engine/msl/ds4_mm.metal+++ b/metal/src/engine/msl/ds4_mm.metal@@@ -1,0 -1,137 +1,138 @@@+ // metal/src/engine/msl/ds4_mm.metal - the DeepSeek prompt chunk's expert GEMMs: moe_mm.metal's mm_gu3 / mm_down3 (matmul2d on dequantized+ // weight tiles) over the engine's own expert slots and its UNPADDED sort order. Compiled after quant/units/mma/moe_mm.metal.+ // tiles[i] = (slot, first row, valid rows): a tile is TM rows of one expert; rows past `valid` (the next expert's) are computed and+ // dropped (the operand rows exist: the scratch carries TM rows of slack), the swiglu is DeepSeek's clamped one.++ struct DsMmArgs {+ ulong gate_off;+ ulong up_off;+ ulong down_off;+ ulong stride; // bytes between slots+ uint gu_row; // bytes per gate/up row (n_embd values)+ uint d_row; // bytes per down row (n_ff values)+ uint n_embd;+ uint n_ff;+ float limit;+ uint pad0;+ };++ // xs[r][c] = half(x[idx[r]][c]) for r < rows+ kernel void ds4_mm_gather(device const float* x [[buffer(0)]], constant ulong& ldx [[buffer(1)]], device const int* idx [[buffer(2)]],+ constant uint& n_embd [[buffer(3)]], device half* xs [[buffer(4)]],+ uint2 tgp [[threadgroup_position_in_grid]], uint tid [[thread_index_in_threadgroup]]) {+ const uint c = tgp.x * 256u + tid;+ if (c >= n_embd) return;+ xs[(ulong) tgp.y * n_embd + c] = half(x[(ulong) idx[tgp.y] * ldx + c]);+ }++ template <int F, int TM, int TN, int TK, int SG>+ kernel void ds4_mm_gu(device const uchar* W [[buffer(0)]], device const half* XS [[buffer(1)]], device half* HS [[buffer(2)]],+ device const int4* tiles [[buffer(3)]], constant DsMmArgs& a [[buffer(4)]], constant uint& n_tiles [[buffer(5)]],+ uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort tid [[thread_index_in_threadgroup]]) {+ if (tg.y >= n_tiles) return;+ threadgroup float ot[SG][TM * TN];+ threadgroup uint tab[fmt<F>::TAB];+ load_tab<F>(tab, tid, SG * 32);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int4 tl = tiles[tg.y];+ const int e = tl.x, row0 = tl.y, valid = tl.z, n0 = (int(tg.x) * SG + sg) * TN;+ device const uchar* E = W + ulong(e) * a.stride;+ device const uchar* g0 = E + a.gate_off + ulong(n0) * a.gu_row;+ device const uchar* u0 = E + a.up_off + ulong(n0) * a.gu_row;+ constexpr auto desc = matmul2d_descriptor(TM, TN, TK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate);+ matmul2d<desc, execution_simdgroup> op;+ tensor<device half, dextents<int32_t, 2>, tensor_inline> tX((device half*)XS, dextents<int32_t, 2>(a.n_embd, 1 << 30));+ auto rg = op.template get_right_input_cooperative_tensor<half, half, float>();+ auto ru = op.template get_right_input_cooperative_tensor<half, half, float>();+ auto tA = tX.template slice<TK, TM>(0, row0);+ auto accg = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rg)>, float>();+ auto accu = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rg)>, float>();+ for (uint16_t i = 0; i < accg.get_capacity(); ++i)+ if (accg.is_valid_element(i)) { accg[i] = 0.0f; accu[i] = 0.0f; }+ for (int k0 = 0; k0 < int(a.n_embd); k0 += TK) {+ const uint16_t cap = rg.get_capacity(), half_cap = cap / 2;+ #pragma clang loop unroll(full)+ for (uint16_t i = 0; i < half_cap; i += 4) {+ const auto ix = rg.get_multidimensional_index(i);+ const int k = k0 + ix[0], n = ix[1];+ device const uchar* rgw = g0 + ulong(n) * a.gu_row;+ device const uchar* ruw = u0 + ulong(n) * a.gu_row;+ half4 wg, wu, wg2, wu2;+ W8<F>::run(rgw, k, tab, wg, wg2);+ W8<F>::run(ruw, k, tab, wu, wu2);+ rg[i] = wg.x; rg[i + 1] = wg.y; rg[i + 2] = wg.z; rg[i + 3] = wg.w;+ ru[i] = wu.x; ru[i + 1] = wu.y; ru[i + 2] = wu.z; ru[i + 3] = wu.w;+ const uint16_t j = i + half_cap;+ rg[j] = wg2.x; rg[j + 1] = wg2.y; rg[j + 2] = wg2.z; rg[j + 3] = wg2.w;+ ru[j] = wu2.x; ru[j + 1] = wu2.y; ru[j + 2] = wu2.z; ru[j + 3] = wu2.w;+ }+ tA = tX.template slice<TK, TM>(k0, row0);+ op.run(tA, rg, accg);+ op.run(tA, ru, accu);+ }+ for (uint16_t i = 0; i < accg.get_capacity(); ++i)+ if (accg.is_valid_element(i)) {+ const float g = fmin(accg[i], a.limit), u = fmin(fmax(accu[i], -a.limit), a.limit);+ accg[i] = g / (1.0f + exp(-g)) * u;+ }+ tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline> tO(ot[sg], dextents<int32_t, 2>(TN, TM));+ accg.store(tO);+ simdgroup_barrier(mem_flags::mem_threadgroup);+ for (int it = lane; it < TM * TN; it += 32) {+ const int m = it / TN, f = it - m * TN;+ if (m < valid) HS[ulong(row0 + m) * a.n_ff + n0 + f] = half(ot[sg][it]);+ }+ }++ template <int F, int TM, int TN, int TK, int SG>+ kernel void ds4_mm_dn(device const uchar* W [[buffer(0)]], device half* YEH [[buffer(1)]], device const half* HS [[buffer(2)]],+ device const int4* tiles [[buffer(3)]], constant DsMmArgs& a [[buffer(4)]], constant uint& n_tiles [[buffer(5)]],+ uint3 tg [[threadgroup_position_in_grid]], ushort sg [[simdgroup_index_in_threadgroup]],+ ushort lane [[thread_index_in_simdgroup]], ushort tid [[thread_index_in_threadgroup]]) {+ if (tg.y >= n_tiles || a.pad0 == 3u) return;+ threadgroup float ot[SG][TM * TN];+ threadgroup uint tab[fmt<F>::TAB];+ load_tab<F>(tab, tid, SG * 32);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ const int4 tl = tiles[tg.y];+ const int e = tl.x, row0 = tl.y, valid = tl.z, n0 = (int(tg.x) * SG + sg) * TN;+ device const uchar* w0 = W + ulong(e) * a.stride + a.down_off + ulong(n0) * a.d_row;+ constexpr auto desc = matmul2d_descriptor(TM, TN, TK, false, true, false, matmul2d_descriptor::mode::multiply_accumulate);+ matmul2d<desc, execution_simdgroup> op;+ tensor<device half, dextents<int32_t, 2>, tensor_inline> tH((device half*)HS, dextents<int32_t, 2>(a.n_ff, 1 << 30));+ auto rw = op.template get_right_input_cooperative_tensor<half, half, float>();+ auto tA = tH.template slice<TK, TM>(0, row0);+ auto acc = op.template get_destination_cooperative_tensor<__tensor_ops_detail::__remove_addrspace_t<decltype(tA)>, __tensor_ops_detail::__remove_addrspace_t<decltype(rw)>, float>();+ for (uint16_t i = 0; i < acc.get_capacity(); ++i)+ if (acc.is_valid_element(i)) acc[i] = 0.0f;+ for (int k0 = 0; k0 < int(a.n_ff); k0 += TK) {+ const uint16_t half_cap = rw.get_capacity() / 2;+ #pragma clang loop unroll(full)+ for (uint16_t i = 0; i < half_cap; i += 4) {+ const auto ix = rw.get_multidimensional_index(i);+ half4 w, w2;+ if (a.pad0 == 1u) { w = half4(0); w2 = half4(0); } else if (a.pad0 == 2u) { w = half4(1); w2 = half4(1); W8<F>::run(w0, 0, tab, w, w2); w = half4(0); } else+ W8<F>::run(w0 + ulong(ix[1]) * a.d_row, k0 + ix[0], tab, w, w2);+ rw[i] = w.x; rw[i + 1] = w.y; rw[i + 2] = w.z; rw[i + 3] = w.w;+ const uint16_t j = i + half_cap;+ rw[j] = w2.x; rw[j + 1] = w2.y; rw[j + 2] = w2.z; rw[j + 3] = w2.w;+ }+ tA = tH.template slice<TK, TM>(k0, row0);+ op.run(tA, rw, acc);+ }+ tensor<threadgroup float, dextents<int32_t, 2>, tensor_inline> tO(ot[sg], dextents<int32_t, 2>(TN, TM));+ acc.store(tO);+ simdgroup_barrier(mem_flags::mem_threadgroup);+ for (int it = lane; it < TM * TN; it += 32) {+ const int m = it / TN, f = it - m * TN;+ if (m < valid) ((device float*) YEH)[ulong(row0 + m) * a.n_embd + n0 + f] = ot[sg][it];+ }+ }+-#define DS4_MM_INST(F, T) template [[host_name("ds4_mm_gu_" #T)]] kernel decltype(ds4_mm_gu<F, 16, 32, 32, 4>) ds4_mm_gu<F, 16, 32, 32, 4>; template [[host_name("ds4_mm_dn_" #T)]] kernel decltype(ds4_mm_dn<F, 16, 32, 32, 4>) ds4_mm_dn<F, 16, 32, 32, 4>;++#define DS4_MM_INST(F, T) template [[host_name("ds4_mm_gu_" #T)]] kernel decltype(ds4_mm_gu<F, 32, 32, 32, 4>) ds4_mm_gu<F, 32, 32, 32, 4>; template [[host_name("ds4_mm_dn_" #T)]] kernel decltype(ds4_mm_dn<F, 32, 32, 32, 4>) ds4_mm_dn<F, 32, 32, 32, 4>;++DS4_MM_INST(T_IQ1_M, 29)+ DS4_MM_INST(T_IQ2_XXS, 16)+ DS4_MM_INST(T_IQ2_XS, 17)+ DS4_MM_INST(T_IQ2_S, 22)+ DS4_MM_INST(T_IQ3_XXS, 18)diff --cc metal/src/mtl_rt.mmindex 0fb436222,4950477bd..9a29db4f1--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@@ -47,8 -67,222 +67,228 @@@ static inline id<MTLBuffer> BO(const Bustatic inline id<MTLComputePipelineState> P(const Pipeline* p) { return (__bridge id<MTLComputePipelineState>)p->handle; }static std::string g_device_name = "unknown GPU";++ // ---- DS4_TIMELINE=1: every deferred dispatch in its own encoder with start / end GPU timestamps; the report at exit gives, per kernel,+ // its GPU time and the gap the GPU sat idle before it (host waits, event waits, launch). DS4_TL_FROM=N: only from dispatch N on.+ static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded+ namespace {+ struct Timeline {+ bool on = false;+ id<MTLCounterSet> tset = nil;+ std::vector<id<MTLCounterSampleBuffer>> bufs; // a sample buffer holds only ~1500 encoders: they rotate+ size_t per = 0, cap = 0, n = 0; // encoders per buffer / in all (the buffers' limit) / recorded+ long after = 0; // DS4_TL_AFTER: record from the Nth dispatch on+ std::vector<std::string> names; // per encoder+ uint64_t cpu0 = 0, gpu0 = 0;+ bool reg = false;+ bool gate = true; // ds4_timeline_gate: record only while open+ bool att = false; // the encoder just made carries a sample slot+ } g_tl;++ id<MTLComputeCommandEncoder> tl_make_encoder(id<MTLDevice> dev, id<MTLCommandBuffer> cb, MTLDispatchType dt) {+ g_tl.att = false;+ if (!g_tl.on) return [cb computeCommandEncoderWithDispatchType:dt];+ if (g_tl.tset == nil) {+ for (id<MTLCounterSet> cs in dev.counterSets)+ if ([cs.name isEqualToString:MTLCommonCounterSetTimestamp]) g_tl.tset = cs;+ if (g_tl.tset == nil) {+ std::fprintf(stderr, "mtl: DS4_TIMELINE: no timestamp counter set\n");+ g_tl.on = false;+ return [cb computeCommandEncoderWithDispatchType:dt];+ }+ for (size_t c = 4096; c >= 64 && g_tl.per == 0; c /= 2) {+ MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];+ d.counterSet = g_tl.tset;+ d.storageMode = MTLStorageModeShared;+ d.sampleCount = c * 2;+ NSError* err = nil;+ id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];+ if (b != nil) {+ g_tl.per = c;+ g_tl.bufs.push_back(b);+ }+ }+ if (g_tl.per == 0) {+ std::fprintf(stderr, "mtl: DS4_TIMELINE: no counter sample buffer\n");+ g_tl.on = false;+ return [cb computeCommandEncoderWithDispatchType:dt];+ }+ g_tl.cap = g_tl.per * 120;+ MTLTimestamp c0 = 0, g0 = 0;+ [dev sampleTimestamps:&c0 gpuTimestamp:&g0];+ g_tl.cpu0 = c0;+ g_tl.gpu0 = g0;+ g_tl.names.assign(g_tl.cap, std::string());+ g_tl.after = std::getenv("DS4_TL_AFTER") ? std::atol(std::getenv("DS4_TL_AFTER")) : 0;+ if (std::getenv("DS4_TL_DRAFT")) g_tl.gate = false;+ }+ if (!g_tl.gate || g_tl.n >= g_tl.cap || (long) g_disp_count.load() < g_tl.after) return [cb computeCommandEncoderWithDispatchType:dt];+ const size_t bi = g_tl.n / g_tl.per, si = g_tl.n % g_tl.per;+ if (bi >= g_tl.bufs.size()) {+ MTLCounterSampleBufferDescriptor* d = [MTLCounterSampleBufferDescriptor new];+ d.counterSet = g_tl.tset;+ d.storageMode = MTLStorageModeShared;+ d.sampleCount = g_tl.per * 2;+ NSError* err = nil;+ id<MTLCounterSampleBuffer> b = [dev newCounterSampleBufferWithDescriptor:d error:&err];+ if (b == nil) {+ g_tl.cap = g_tl.n;+ return [cb computeCommandEncoderWithDispatchType:dt];+ }+ g_tl.bufs.push_back(b);+ }+ MTLComputePassDescriptor* pd = [MTLComputePassDescriptor computePassDescriptor];+ pd.dispatchType = dt;+ pd.sampleBufferAttachments[0].sampleBuffer = g_tl.bufs[bi];+ pd.sampleBufferAttachments[0].startOfEncoderSampleIndex = 2 * si;+ pd.sampleBufferAttachments[0].endOfEncoderSampleIndex = 2 * si + 1;+ g_tl.att = true;+ return [cb computeCommandEncoderWithDescriptor:pd];+ }++ void tl_report(id<MTLDevice> dev) {+ if (!g_tl.on && g_tl.bufs.empty()) return;+ const size_t n = std::min(g_tl.n, g_tl.cap);+ if (n < 2) return;+ std::vector<MTLCounterResultTimestamp> tsv(2 * n);+ for (size_t b = 0; b * g_tl.per < n; ++b) {+ const size_t cnt = std::min(g_tl.per, n - b * g_tl.per);+ NSData* data = [g_tl.bufs[b] resolveCounterRange:NSMakeRange(0, 2 * cnt)];+ if (data == nil) { std::fprintf(stderr, "mtl: DS4_TIMELINE: resolve failed\n"); return; }+ std::memcpy(&tsv[2 * b * g_tl.per], [data bytes], 2 * cnt * sizeof(MTLCounterResultTimestamp));+ }+ const MTLCounterResultTimestamp* ts = tsv.data();+ MTLTimestamp c1 = 0, g1 = 0;+ [dev sampleTimestamps:&c1 gpuTimestamp:&g1];+ mach_timebase_info_data_t tb;+ mach_timebase_info(&tb);+ const double cpu_ns_per_tick = (double) tb.numer / (double) tb.denom;+ (void) cpu_ns_per_tick; // (sampleTimestamps' CPU value is in ns already)+ const double ns_per_gpu = (g1 > g_tl.gpu0) ? (double) (c1 - g_tl.cpu0) / (double) (g1 - g_tl.gpu0) : 1.0;+ std::fprintf(stderr, "timeline: cap %zu n %zu scale %.6f ns/tick first ts %llu %llu last %llu\n", g_tl.cap, g_tl.n, ns_per_gpu, (unsigned long long) ts[0].timestamp, (unsigned long long) ts[1].timestamp, (unsigned long long) ts[2 * n - 1].timestamp);+ size_t from = std::getenv("DS4_TL_FROM") ? (size_t) std::atoll(std::getenv("DS4_TL_FROM")) : 0;+ struct Acc { double busy = 0, gap = 0, gap_big = 0; size_t n = 0, n_big = 0; };+ std::map<std::string, Acc> acc;+ double t_first = 0, t_last = 0, total_busy = 0, total_gap = 0, total_big = 0;+ double prev_end = -1, ovl_us = 0;+ size_t used = 0, n_ovl = 0;+ for (size_t k = from; k < n; ++k) {+ if (g_tl.names[k].empty()) continue;+ const double st = (double) ts[2 * k].timestamp * ns_per_gpu * 1e-3, en = (double) ts[2 * k + 1].timestamp * ns_per_gpu * 1e-3; // us+ if (!(en >= st)) continue;+ if (prev_end < 0) t_first = st;+ Acc& a = acc[g_tl.names[k]];+ a.busy += en - st;+ ++a.n;+ total_busy += en - st;+ if (prev_end >= 0 && st < prev_end) { ++n_ovl; ovl_us += prev_end - st; }+ if (prev_end >= 0 && st >= prev_end) {+ const double gp = st - prev_end;+ a.gap += gp;+ total_gap += gp;+ if (gp > 300.0) { a.gap_big += gp; ++a.n_big; total_big += gp; }+ }+ prev_end = en;+ t_last = en;+ ++used;+ }+ std::vector<std::pair<double, std::string>> v;+ for (auto& kv : acc) v.emplace_back(kv.second.busy + kv.second.gap, kv.first);+ std::sort(v.begin(), v.end(), [](auto& x, auto& y) { return x.first > y.first; });+ std::fprintf(stderr, "timeline: %zu dispatches, wall %.1f ms, GPU busy %.1f ms, idle gaps %.1f ms (of it > 300 us: %.1f ms)\n", used,+ (t_last - t_first) / 1000.0, total_busy / 1000.0, total_gap / 1000.0, total_big / 1000.0);+ std::fprintf(stderr, "timeline: %zu dispatches started before the previous one ended (%.1f ms overlapped)\n", n_ovl, ovl_us / 1000.0);+ std::fprintf(stderr, "timeline: kernel n busy ms us/disp gap ms gap/disp big-gap ms (n)\n");+ for (size_t i = 0; i < v.size() && i < 40; ++i) {+ const Acc& a = acc[v[i].second];+ std::fprintf(stderr, "timeline: %-28s %6zu %9.1f %8.1f %8.1f %8.1f %10.1f (%zu)\n", v[i].second.c_str(), a.n, a.busy / 1000.0, a.busy / (double) a.n,+ a.gap / 1000.0, a.gap / (double) a.n, a.gap_big / 1000.0, a.n_big);+ }+ }+ } // namespace++ extern "C" void ds4_timeline_gate(int open) { g_tl.gate = open != 0; }+++ // ---- ds4_busy_begin / ds4_busy_end: a one-threadgroup kernel on a second queue that spins on a shared flag while the host works+ // (the CPU lane), so the GPU is not idle between two bursts (idle -> the next kernels run 2-3x slower); the host sets the flag the+ // moment it is done and the kernel leaves within microseconds. DS4_BUSY=0 turns it off.+ namespace {+ struct Busy {+ bool init = false, ok = false;+ id<MTLCommandQueue> q = nil;+ id<MTLComputePipelineState> ps = nil;+ id<MTLBuffer> flag = nil;+ id<MTLCommandBuffer> last = nil;+ bool running = false;+ };+ Busy g_busy;+ } // namespace++ extern "C" void ds4_busy_begin() {+ static const bool off = std::getenv("DS4_BUSY") && std::string(std::getenv("DS4_BUSY")) == "0";+ if (off) return;+ @autoreleasepool {+ if (!g_busy.init) {+ g_busy.init = true;+ id<MTLDevice> dev = MTLCreateSystemDefaultDevice();+ static const char* kSrc = R"MSL(+ #include <metal_stdlib>+ using namespace metal;+ kernel void busy_spin(device volatile atomic_uint* flag [[buffer(0)]], constant uint& cap [[buffer(1)]], uint t [[thread_position_in_grid]]) {+ uint x = t;++ float f = (float) t * 1e-9f;+ for (uint i = 0; i < cap; ++i) {+ for (uint j = 0; j < 64; ++j) x = x * 1664525u + 1013904223u;++ for (uint j = 0; j < 16; ++j) f = fma(f, 1.0000001f, 1e-7f); // (a real FP chain: keeps the ALU clock up)++ x ^= as_type<uint>(f);+ if (atomic_load_explicit(flag, memory_order_relaxed) != 0u) break;+ }+ if (x == 0xdeadbeefu) atomic_store_explicit(flag, 2u, memory_order_relaxed);+ }+ )MSL";+ NSError* err = nil;+ id<MTLLibrary> lib = [dev newLibraryWithSource:[NSString stringWithUTF8String:kSrc] options:nil error:&err];+ if (lib) g_busy.ps = [dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"busy_spin"] error:&err];+ if (g_busy.ps) {+ g_busy.q = [dev newCommandQueue];+ g_busy.flag = [dev newBufferWithLength:64 options:MTLResourceStorageModeShared];+ g_busy.ok = true;+ }+ }+ if (!g_busy.ok || g_busy.running) return;+ g_busy.running = true;+ if (g_busy.last != nil) [g_busy.last waitUntilCompleted]; // (the previous spin has left)+ *(volatile uint32_t*) [g_busy.flag contents] = 0;+ uint32_t cap = std::getenv("DS4_BUSY_CAP") ? (uint32_t) std::atoi(std::getenv("DS4_BUSY_CAP")) : 20000u; // ~ tens of ms at most++ // the spin's width: one 256-thread threadgroup leaves most of the GPU idle and the clock drops anyway; DS4_BUSY_W++ // threadgroups (256 threads each) keep more of the card's ALUs fed, which is what holds the performance state.++ static const uint32_t width = std::getenv("DS4_BUSY_W") ? (uint32_t) std::max(1, std::atoi(std::getenv("DS4_BUSY_W"))) : 1u;+ id<MTLCommandBuffer> cb = [g_busy.q commandBuffer];+ id<MTLComputeCommandEncoder> enc = [cb computeCommandEncoder];+ [enc setComputePipelineState:g_busy.ps];+ [enc setBuffer:g_busy.flag offset:0 atIndex:0];+ [enc setBytes:&cap length:4 atIndex:1];- [enc dispatchThreads:MTLSizeMake(256, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];++ [enc dispatchThreads:MTLSizeMake(256 * width, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];+ [enc endEncoding];+ [cb commit];+ g_busy.last = cb;+ }+ }+ extern "C" void ds4_busy_end() {+ if (!g_busy.ok || g_busy.last == nil || !g_busy.running) return;+ *(volatile uint32_t*) [g_busy.flag contents] = 1;+ g_busy.running = false;+ }+Context::Context() : impl_(new Impl) {impl_->dev = MTLCreateSystemDefaultDevice();+ if (std::getenv("DS4_TIMELINE") && !g_tl.reg) {+ g_tl.on = true;+ g_tl.reg = true;+ static id<MTLDevice> tl_dev = impl_->dev;+ std::atexit([] { tl_report(tl_dev); });+ }if (!impl_->dev) {std::fprintf(stderr, "mtl: no Metal device\n");std::exit(2);@@@ -249,9 -616,12 +622,17 @@@ void Context::free(Buffer& b)b = Buffer{};}+ static std::mutex g_hist_mu;+ static std::unordered_map<void*, std::string> g_pipe_names;+ static std::unordered_map<std::string, uint64_t> g_kernel_counts;++// DS4_ENC_TIMES: where a dispatch's encoding time goes - the pipeline state, the arguments, the dispatch call++static std::atomic<uint64_t> g_t_pipe{0}, g_t_args{0}, g_t_disp{0};static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {require_fit(d, "dispatch");++ static const bool enc_times = std::getenv("DS4_ENC_TIMES") != nullptr;++ const auto t_a = std::chrono::steady_clock::now();[enc setComputePipelineState:P(d.p)];++ const auto t_b = std::chrono::steady_clock::now();for (size_t i = 0; i < d.args.size(); ++i) {const Arg& a = d.args[i];if (a.buf) {@@@ -262,6 -634,6 +645,7 @@@else[enc setBytes:a.bytes.data() length:a.bytes.size() atIndex:i];}++ const auto t_c = std::chrono::steady_clock::now();if (d.tg_mem) [enc setThreadgroupMemoryLength:(d.tg_mem + 15) / 16 * 16 atIndex:0];if (d.indirect)[enc dispatchThreadgroupsWithIndirectBuffer:B(*d.indirect)@@@ -270,10 -642,187 +654,212 @@@else[enc dispatchThreadgroups:MTLSizeMake(d.grid[0], d.grid[1], d.grid[2])threadsPerThreadgroup:MTLSizeMake(d.tg[0], d.tg[1], d.tg[2])];++ if (enc_times) {++ const auto t_d = std::chrono::steady_clock::now();++ g_t_pipe.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_b - t_a).count(), std::memory_order_relaxed);++ g_t_args.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_c - t_b).count(), std::memory_order_relaxed);++ g_t_disp.fetch_add((uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(t_d - t_c).count(), std::memory_order_relaxed);++ }+ }++ static std::deque<std::string> g_recent; // the last dispatches encoded (printed with a command buffer failure)+ static std::string recent_str() { std::string r; for (auto& n : g_recent) r += n + " "; return r; }+ static double g_gpu_ms = 0.0;+ static std::atomic<uint64_t> g_gpu_busy_us{0}; // DS4_GPU_TOTAL: the GPU time of every committed command buffer+ static void add_busy_handler(id<MTLCommandBuffer> cb) {+ static const bool on = std::getenv("DS4_GPU_TOTAL") != nullptr;+ if (on) [cb addCompletedHandler:^(id<MTLCommandBuffer> c) { const double d = c.GPUEndTime - c.GPUStartTime; if (d > 0) g_gpu_busy_us += (uint64_t) (d * 1e6); }];+ } // GPU time of every committed command buffer, summed+ static std::atomic<uint64_t> g_wait_ns{0}, g_enc_ns{0}, g_wait_n{0}; // host time: waiting on command buffers; encoding dispatches+ static std::mutex g_prof_mu;+ static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)+ static std::string g_prof_last;+ static std::atomic<uint64_t> g_cb_count{0};+ static bool g_count_reg = false;+ static void ds4_report_counts() {+ if (g_gpu_busy_us.load()) std::fprintf(stderr, "mtl: GPU busy (every command buffer) %.1f ms\n", g_gpu_busy_us.load() / 1000.0);+ std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,+ (unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());+ std::fprintf(stderr, "mtl: host waited %.1f ms for the GPU in %llu waits; encoding the dispatches took %.1f ms\n", g_wait_ns.load() / 1e6, (unsigned long long) g_wait_n.load(), g_enc_ns.load() / 1e6);++ if (g_t_pipe.load() || g_t_args.load() || g_t_disp.load())++ std::fprintf(stderr, "mtl: of the encoding: pipeline %.1f ms, arguments %.1f ms, dispatch %.1f ms\n",++ g_t_pipe.load() / 1e6, g_t_args.load() / 1e6, g_t_disp.load() / 1e6);+ if (!g_prof.empty()) {+ std::vector<std::pair<double, std::string>> pv;+ double tot = 0;+ for (auto& kv : g_prof) { pv.emplace_back(kv.second.first, kv.first); tot += kv.second.first; }+ std::sort(pv.begin(), pv.end(), [](auto& a, auto& b) { return a.first > b.first; });+ std::fprintf(stderr, "prof: kernel GPU ms n us/dispatch (total %.1f ms)\n", tot / 1000.0);+ for (size_t i = 0; i < pv.size() && i < 30; ++i) {+ auto& e = g_prof[pv[i].second];+ std::fprintf(stderr, "prof: %-30s %8.1f %6llu %8.1f\n", pv[i].second.c_str(), e.first / 1000.0, (unsigned long long) e.second, e.first / (double) e.second);+ }+ }+ std::vector<std::pair<uint64_t, std::string>> v;+ { std::lock_guard<std::mutex> lk(g_hist_mu); for (auto& kv : g_kernel_counts) v.emplace_back(kv.second, kv.first); }+ std::sort(v.begin(), v.end(), [](auto& a, auto& b) { return a.first > b.first; });+ for (size_t i = 0; i < v.size() && i < 14; ++i)+ std::fprintf(stderr, " %6llu %s\n", (unsigned long long) v[i].first, v[i].second.c_str());+ }++ extern "C" unsigned long long ds4_dispatch_seq() { return g_disp_count.load(); }++ extern "C" void ds4_gpu_sync_all_streams_weak() __asm("_ds4_gpu_sync_all_streams") __attribute__((weak_import));+ void Context::split_runs() {+ // (the streams' pending copies come first, as in a full flush: they are queued on another MTLCommandQueue and would+ // otherwise not have run when the commands committed here read what they write)+ if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();+ // Commit the batched dispatches as their own command buffer WITHOUT waiting: the queue runs buffers in order and makes each+ // one's writes visible to the next. (Measured: a producer kernel's output was read stale by the next dispatch of the same+ // buffer for two kernels in a layer; ending the buffer after them is enough, no wait needed.)+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);+ if (impl_->cb_def == nil) return;+ ++g_cb_count;+ [impl_->enc_def endEncoding];+ if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];+ [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];+ add_busy_handler(impl_->cb_def);+ [impl_->cb_def commit];+ impl_->inflight = impl_->cb_def;+ impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();+ impl_->cb_def = nil;+ impl_->enc_def = nil;+ impl_->def_n = 0;+ }++ // Waiting for a committed command buffer: spin on its status for a while (the thread wake of waitUntilCompleted costs+ // ~1 ms a wait; the decode token waits ~100 times), then block. DS4_NO_SPIN=1: always block.+ static void wait_spin(id<MTLCommandBuffer> cb) {+ static const bool no_spin = std::getenv("DS4_NO_SPIN") != nullptr;+ static const double budget = std::getenv("DS4_SPIN_US") ? std::atof(std::getenv("DS4_SPIN_US")) * 1e-6 : 20e-3;+ if (!no_spin) {+ const auto t0 = std::chrono::steady_clock::now();+ while (cb.status < MTLCommandBufferStatusCompleted) {+ if (std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count() > budget) break;+ }+ }+ if (cb.status < MTLCommandBufferStatusCompleted) [cb waitUntilCompleted];+ }++ void* Context::mark() {+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);+ if (impl_->cb_def != nil) split_runs(); // (the streams' pending work goes first, inside)+ if (impl_->inflight == nil) return nullptr;+ return (__bridge_retained void*) impl_->inflight;+ }++ void Context::wait_mark(void* token) {+ id<MTLCommandBuffer> cb = (__bridge_transfer id<MTLCommandBuffer>) token;+ const auto tw0 = std::chrono::steady_clock::now();+ wait_spin(cb);+ g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();+ ++g_wait_n;+ if (cb.status != MTLCommandBufferStatusCompleted)+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[cb.error localizedDescription] UTF8String]);+ if (cb.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: recent: %s\n", recent_str().c_str()), std::fprintf(stderr, "mtl: info: %s\n", [[[cb.error userInfo] description] UTF8String]);+ }++ void Context::flush_runs() {+ if (std::getenv("DS4_TRACE_DISP") && impl_->cb_def != nil) std::fprintf(stderr, "TRACE flush_runs (%d queued)\n", impl_->def_n);++ static const bool cb_log = std::getenv("DS4_CB_LOG") != nullptr; // debugging: one line a command buffer (what it held)++ if (cb_log && impl_->cb_def != nil) {++ std::string first, last;++ if (!impl_->def_names.empty()) {++ size_t sp = impl_->def_names.find(' ');++ first = impl_->def_names.substr(0, sp);++ size_t e = impl_->def_names.find_last_not_of(' ');++ size_t b = impl_->def_names.find_last_of(' ', e);++ last = impl_->def_names.substr(b == std::string::npos ? 0 : b + 1, e - (b == std::string::npos ? 0 : b));++ }++ std::fprintf(stderr, "CB %d disp first=%-24s last=%s\n", impl_->def_n, first.c_str(), last.c_str());++ }+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);+ if (!g_count_reg) { g_count_reg = true; std::atexit(ds4_report_counts); }+ if (impl_->cb_def != nil) {+ ++g_cb_count;+ [impl_->enc_def endEncoding];+ if (impl_->qev == nil) impl_->qev = [impl_->dev newSharedEvent];+ [impl_->cb_def encodeSignalEvent:impl_->qev value:++impl_->qev_v];+ add_busy_handler(impl_->cb_def);+ [impl_->cb_def commit];+ impl_->inflight = impl_->cb_def;+ impl_->inflight_names = std::move(impl_->def_names); impl_->def_names.clear();+ impl_->cb_def = nil;+ impl_->enc_def = nil;+ impl_->def_n = 0;+ }+ if (impl_->inflight != nil) {+ if (std::getenv("DS4_TRACE_WAIT")) { void* fr[8]; const int nf = backtrace(fr, 8); std::fprintf(stderr, "WAIT flush_runs:\n"); backtrace_symbols_fd(fr + 1, nf - 1, 2); }+ const auto tw0 = std::chrono::steady_clock::now();+ wait_spin(impl_->inflight);+ g_wait_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - tw0).count();+ ++g_wait_n;+ { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;+ if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }+ if (impl_->inflight.status != MTLCommandBufferStatusCompleted)+ std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);+ if (impl_->inflight.status != MTLCommandBufferStatusCompleted && std::getenv("DS4_ERRINFO")) std::fprintf(stderr, "mtl: failed cb held: %s\n", impl_->inflight_names.c_str());+ impl_->inflight = nil;+ }+ for (void* h : impl_->graveyard) { id<MTLBuffer> x = (__bridge_transfer id<MTLBuffer>)h; x = nil; }+ impl_->graveyard.clear();}double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent, bool barrier_between) {+ std::lock_guard<std::recursive_mutex> lk_(impl_->mu);@autoreleasepool {+ if (ds.size() == 1 && reps == 1 && !barrier_between) {+ { static const std::string skipk = std::getenv("DS4_SKIP_K") ? std::string(",") + std::getenv("DS4_SKIP_K") + "," : std::string(); // debugging: drop these kernels (timing only)+ if (!skipk.empty() && skipk.find("," + ds[0].p->name + ",") != std::string::npos) return 0.0; }+ if (impl_->cb_def == nil || impl_->def_n >= 2000000) {+ if (impl_->cb_def != nil) split_runs(); // (a buffer committed earlier may still be running: the queue keeps the order)+ if (std::getenv("DS4_ERRINFO")) {+ MTLCommandBufferDescriptor* cd = [MTLCommandBufferDescriptor new];+ cd.retainedReferences = NO;+ cd.errorOptions = MTLCommandBufferErrorOptionEncoderExecutionStatus;+ impl_->cb_def = [impl_->q commandBufferWithDescriptor:cd];+ } else+ impl_->cb_def = [impl_->q commandBufferWithUnretainedReferences];- if (impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];++ // (the wait on our own event is only for a *different* queue: this queue already runs the previous buffer++ // before this one, and the wait costs a full GPU round trip at every buffer - 107 buffers a decode token,++ // 0.28 ms each. DS4_QEV_WAIT=1 restores it.)++ static const bool qev_wait = std::getenv("DS4_NO_QEV_WAIT") == nullptr; // (measured: off is slower - keep it)++ if (qev_wait && impl_->qev != nil && impl_->qev_v > 0) [impl_->cb_def encodeWaitForEvent:impl_->qev value:impl_->qev_v];+ impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial);+ }+ const auto te0 = std::chrono::steady_clock::now();+ encode(impl_->enc_def, ds[0]);+ if (g_tl.on && g_tl.att && g_tl.n < g_tl.cap) { g_tl.names[g_tl.n++] = ds[0].p->name; g_tl.att = false; }+ g_last_act_ns.store(ka_now_ns(), std::memory_order_relaxed);+ g_recent.push_back(ds[0].p->name); if (g_recent.size() > 8) g_recent.pop_front();+ if (std::getenv("DS4_ERRINFO")) impl_->def_names += ds[0].p->name + " ";+ g_enc_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - te0).count();+ static const bool trace = std::getenv("DS4_TRACE_DISP") != nullptr;+ if (trace) { std::lock_guard<std::mutex> lk(g_hist_mu); std::fprintf(stderr, "TRACE disp %s\n", ds[0].p->name.c_str()); }+ // Every wrapper binds its own MTLBuffer over the same memory, so Metal's hazard tracking cannot see that one+ // dispatch reads what the previous wrote: without an explicit barrier the reader saw the stale value (measured: the+ // second matvec's activation quantiser read the previous layer's x).+ static const bool no_mb = std::getenv("DS4_NO_MBARRIER") != nullptr;+ if (!no_mb) [impl_->enc_def memoryBarrierWithScope:MTLBarrierScopeBuffers];+ static const bool enc_each = std::getenv("DS4_ENC_EACH") != nullptr || g_tl.on; // a new encoder per dispatch (debugging, the timeline)+ if (enc_each) {+ [impl_->enc_def endEncoding];+ impl_->enc_def = tl_make_encoder(impl_->dev, impl_->cb_def, (std::getenv("DS4_CONCURRENT") ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial));+ }+ ++impl_->def_n;+ ++g_disp_count;+ static const bool prof = std::getenv("DS4_PROF") != nullptr; // every dispatch alone, its GPU time kept per kernel+ if (prof) { g_prof_last = ds[0].p->name; if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak(); flush_runs(); return 0.0; }+ static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr; // debugging: every dispatch alone, in order+ static const std::string only = std::getenv("DS4_FLUSH_AFTER") ? std::getenv("DS4_FLUSH_AFTER") : ""; // ",name1,name2,"+ const bool listed = !only.empty() && only.find("," + ds[0].p->name + ",") != std::string::npos;+ if (each || listed) {+ if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak();+ flush_runs();+ }+ return 0.0;+ }+ flush_runs();id<MTLCommandBuffer> cb = [impl_->q commandBufferWithUnretainedReferences];id<MTLComputeCommandEncoder> enc =[cb computeCommandEncoderWithDispatchType:concurrent ? MTLDispatchTypeConcurrent : MTLDispatchTypeSerial];diff --cc packaging/mac/build_dsv41_engine.shindex 000000000,000000000..1415a2078new file mode 100644--- /dev/null+++ b/packaging/mac/build_dsv41_engine.sh@@@ -1,0 -1,0 +1,58 @@@++#!/bin/bash++# Build the DeepSeek-V4.1-Flash engine component for Apple Silicon (strata_ds4, backend metal).++#++# bash packaging/mac/build_dsv41_engine.sh [--version 0.2.7] [--build DIR] [--out DIR]++#++# Run it in the source tree on the Mac, after `ninja -C <build> strata_ds4` (the script builds it too when the++# binary is missing). It stages the binary and the kernel sources the engine compiles at start-up, packages them with++# packaging/package_engine.py and verifies every hash. It does NOT sign: the feed key stays offline, and a package++# made here is unsigned and loads (see packaging/ENGINE-COMPONENT.md, Signature).++#++# Layout (strata_ds4 finds these beside its own executable, metal/src/resource_path.h):++# <exe>/metal/kernels/*.metal the shared Qwen kernels the DeepSeek host also compiles++# <exe>/metal/src/engine/msl/*.metal ds4_init, ds4_iq, ds4_mm, ds4_quant, mtp, mtp_chain, ple++# <exe>/metal/ggml-common.h++set -u++ROOT=$(cd "$(dirname "$0")/../.." && pwd)++VERSION=0.2.7++BUILD="$ROOT/build-metal"++OUT="$ROOT/pkg/engine"++while [ $# -gt 0 ]; do++ case "$1" in++ --version) VERSION="$2"; shift 2;;++ --build) BUILD="$2"; shift 2;;++ --out) OUT="$2"; shift 2;;++ *) echo "usage: $0 [--version V] [--build DIR] [--out DIR]"; exit 2;;++ esac++done++NINJA=${NINJA:-$HOME/strata-mac/tools/bin/ninja}++EXE="$BUILD/strata_ds4"++[ -x "$EXE" ] || "$NINJA" -C "$BUILD" strata_ds4 || exit 1++++STAGE="$ROOT/pkg/dsv41-stage"++rm -rf "$STAGE"++mkdir -p "$STAGE/metal/kernels" "$STAGE/metal/src/engine/msl"++cp "$EXE" "$STAGE/"; chmod +x "$STAGE/strata_ds4"++cp "$ROOT"/metal/kernels/*.metal "$STAGE/metal/kernels/"++for f in ds4_init ds4_iq ds4_mm ds4_quant mtp mtp_chain ple; do++ cp "$ROOT/metal/src/engine/msl/$f.metal" "$STAGE/metal/src/engine/msl/" || exit 1++done++cp "$ROOT/third_party/ggml/ggml-common.h" "$STAGE/metal/"++++python3 "$ROOT/packaging/package_engine.py" --src "$STAGE" --backend metal --version "$VERSION" \++ --platform macos-arm64 --arch apple-gpu --entrypoint strata_ds4 --include 'metal/**' --out "$OUT" || exit 1++DIR="$OUT/$VERSION-macos-arm64-metal"++python3 "$ROOT/packaging/package_engine.py" --verify "$DIR" || exit 1++++# self-containment: the package must run with the source tree's kernels hidden++PACK=${PACK:-$HOME/strata-mac/pack-2.25-real}++if [ ! -d "$PACK" ]; then echo "== no pack at $PACK: package built, smoke test skipped"; exit 0; fi++echo "== self-containment test (source kernels hidden, run from /tmp)"++mv "$ROOT/metal/kernels" "$ROOT/metal/kernels.hidden"++mv "$ROOT/metal/src/engine/msl" "$ROOT/metal/src/engine/msl.hidden"++( cd /tmp && "$DIR/strata_ds4" --pack "$PACK" --ids 0 --n 1 \++ --expert-profile none --vision off --dspark off --cpu-threads 4 --threads off --tp 1 2>&1 | grep -a -E 'prompt:|decode:|cannot read|failed' | head -3 )++rc=$?++mv "$ROOT/metal/kernels.hidden" "$ROOT/metal/kernels"++mv "$ROOT/metal/src/engine/msl.hidden" "$ROOT/metal/src/engine/msl"++echo "== done: $DIR (rc=$rc)"diff --cc src/dsv41/ds4_attn_split.cuhindex 959830b09,9def53b8c..7d9028dfd--- a/src/dsv41/ds4_attn_split.cuh+++ b/src/dsv41/ds4_attn_split.cuh@@@ -11,6 -11,6 +11,7 @@@// phase 2: block (head, query): merge the chunks and the sink.// Plain C++ (no PTX), shuffles and shared memory only: the HIP port is a rename.#pragma once++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "ds4_kernels.cuh"#include "ds4_numerics.cuh"#include <cuda_fp16.h>diff --cc src/dsv41/ds4_gguf.cuindex 55231c8d3,ba4ed3e3d..fe29d2034--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@@ -1,9 -1,13 +1,14 @@@--// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).++// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.// src/ds4/ds4_gguf.cu - a GGUF's BF16 matrices and MXFP4 experts on the device (see ds4_gguf.cuh).++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "ds4_gguf.cuh"+ #if defined(STRATA_METAL)+ #include <ds4_metal_host.h>+ #endif#include "ds4_iq.cuh"+ #include "ds4_gguf_cpu.hpp"#include "ds4_numerics.cuh"#include <cuda_fp16.h>@@@ -198,65 -202,12 +203,77 @@@ __global__ void __launch_bounds__(kThre}void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d, cudaStream_t s) {const dim3 grid((unsigned) ((k / 32 + kWarps - 1) / kWarps), (unsigned) rows);- q8_quant_kernel<<<grid, kThreads, 0, s>>>(x, ldx, k, q, d);+ #if defined(STRATA_METAL)+ if (::strata::dsv41::metal_host::q8_quant_rows(x, ldx, rows, k, (int8_t*) q, d)) return;+ #endif+ strata_ds4_launch(q8_quant_kernel, grid, kThreads, 0, s, x, ldx, k, q, d);}+// Several activations' rounding in one launch (a GEMV call's jobs: wo_a's eight groups, the shared expert's two, ...): item z is+// q8_quant's (x, ldx, rows, k, q, d), the same per-element arithmetic, so the bytes are the single launches'. Identical items+// (the same x, stride, rows and k: two matrices reading one input) are rounded once and share the result.+constexpr int kQuantItems = x3::kMaxMvJobs;+struct QuantItems {+ const float* x[kQuantItems];+ size_t ldx[kQuantItems];+ int k[kQuantItems], rows[kQuantItems];+ int8_t* q[kQuantItems];+ float* d[kQuantItems];+ int n;+};+__global__ void __launch_bounds__(kThreads) q8_quant_multi_kernel(QuantItems it) {+ const int z = blockIdx.z;+ if (z >= it.n) return;+ const int r = blockIdx.y, b = blockIdx.x * kWarps + (threadIdx.x >> 5), lane = threadIdx.x & 31;+ const int k = it.k[z];+ if (r >= it.rows[z] || b * 32 >= k) return;+ const float v = it.x[z][(size_t) r * it.ldx[z] + (size_t) b * 32 + lane];+ float am = fabsf(v);+#pragma unroll+ for (int o = 16; o > 0; o >>= 1) am = fmaxf(am, __shfl_xor_sync(0xffffffffu, am, o));+ const float sc = am / 127.0f;+ it.q[z][(size_t) r * k + (size_t) b * 32 + lane] = (int8_t) (am > 0.0f ? __float2int_rn(v / sc) : 0);+ if (lane == 0) it.d[z][(size_t) r * (k / 32) + b] = sc;+}+// adds an item unless an identical one is queued; returns the item whose result to read (its q and d)+int quant_add(QuantItems& it, const float* x, size_t ldx, int rows, int k, int8_t* q, float* d) {+ static const bool nodedup = std::getenv("DS4_QUANT_NODEDUP") != nullptr;+ for (int i = 0; i < it.n && !nodedup; ++i)+ if (it.x[i] == x && it.ldx[i] == ldx && it.rows[i] == rows && it.k[i] == k) return i;+ const int i = it.n++;+ it.x[i] = x;+ it.ldx[i] = ldx;+ it.rows[i] = rows;+ it.k[i] = k;+ it.q[i] = q;+ it.d[i] = d;+ return i;+}+void quant_flush(const QuantItems& it, cudaStream_t s) {+ if (it.n <= 0) return;+ static const bool single = std::getenv("DS4_QUANT_SINGLE") != nullptr;++#if defined(STRATA_METAL)++ if (!single && std::getenv("DS4_NO_Q8_BATCH") == nullptr) { // the call's roundings in one dispatch (a dispatch a job was 45% of the decode token's)++ struct Spec { const float* x; uint64_t ldx; uint32_t rows, k; int8_t* q; float* d; } sp[kQuantItems];++ for (int i = 0; i < it.n; ++i) sp[i] = Spec{it.x[i], (uint64_t) it.ldx[i], (uint32_t) it.rows[i], (uint32_t) it.k[i], it.q[i], it.d[i]};++ if (::strata::dsv41::metal_host::q8_quant_multi(sp, it.n)) return;++ }++ for (int i = 0; i < it.n; ++i) q8_quant(it.x[i], it.ldx[i], it.rows[i], it.k[i], it.q[i], it.d[i], s);++ return;++#endif+ if (single) {+ for (int i = 0; i < it.n; ++i) q8_quant(it.x[i], it.ldx[i], it.rows[i], it.k[i], it.q[i], it.d[i], s);+ return;+ }+ int kmax = 0, rmax = 0;+ for (int i = 0; i < it.n; ++i) {+ kmax = std::max(kmax, it.k[i]);+ rmax = std::max(rmax, it.rows[i]);+ }+ const dim3 grid((unsigned) ((kmax / 32 + kWarps - 1) / kWarps), (unsigned) rmax, (unsigned) it.n);- q8_quant_multi_kernel<<<grid, kThreads, 0, s>>>(it);++ strata_ds4_launch(q8_quant_multi_kernel, grid, kThreads, 0, s, it);+}+// 8 E2M1 codes (a word of 4 code bytes) -> their values as int8: .x the low nibbles' (values 0..3 of the word's bytes),// .y the high nibbles' - llama.cpp's get_int_from_table_16 (MIT), the table kvalues_mxfp4__device__ __forceinline__ int2 e2m1_x8(uint32_t q4) {@@@ -1064,6 -1015,6 +1081,14 @@@ size_t type_row_bytes(int type, int64_int type_block(int type) { return type == kTypeMXFP4 ? kMxBlock : iq::known(type) ? iq::block_values(type) : 0; }const char* type_name(int type) { return type == kTypeMXFP4 ? "MXFP4" : iq::name(type); }bool type_on_device(int type) { return type == kTypeMXFP4 || iq::supported(type); }++bool type_on_chunk(int type) {++#if defined(STRATA_METAL)++ // ds4_mm.metal instantiates the tile GEMM for IQ2_XXS / IQ2_XS / IQ2_S only; IQ1_M (the 2.00 bpw pack's type) has++ // the matvec kernel but no tile one, so a chunked prompt would stop in "kernel launch with no Metal dispatch".++ if (type == iq::kIQ1_S || type == iq::kIQ4_NL) return false; // (IQ1_M: the tile kernels since DS4_MM_INST(29) and iq_bs)++#endif++ return type_on_device(type);++}namespace {size_t row_stride(const x3::Mat& m) { // values (BF16) or bytes (MXFP4, Q8_0, a low-bit type) a row@@@ -1128,6 -1093,15 +1167,23 @@@ x3::Mat view(const x3::Mat& m, int c0,bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {if (n <= 0 || n > x3::kMaxMvJobs) return false;+ #if defined(STRATA_METAL)- // the jobs' int8 activations are quantised in one dispatch before the matvecs (a dispatch a job was 45% of the decode token's)- struct Q8Batch { const float* x; uint64_t ldx; uint32_t rows, k; int8_t* q; float* d; } q8_batch[x3::kMaxMvJobs];- int nq8 = 0;- const bool q8_deferred = std::getenv("DS4_NO_Q8_BATCH") == nullptr && std::getenv("DS4_DBG_Q8") == nullptr;-#define Q8_QUANT(XF, LDX, ROWS, KK, Q, D, S) do { if (q8_deferred) q8_batch[nq8++] = Q8Batch{XF, (uint64_t) (LDX), (uint32_t) (ROWS), (uint32_t) (KK), Q, D}; else q8_quant(XF, LDX, ROWS, KK, Q, D, S); } while (0)-#else-#define Q8_QUANT(XF, LDX, ROWS, KK, Q, D, S) q8_quant(XF, LDX, ROWS, KK, Q, D, S)++ if (std::getenv("DS4_MV_LOG")) { // how the engine batches its matvec jobs (one call = one quantiser dispatch)++ static std::atomic<unsigned long long> calls{0}, jobsum{0};++ static std::atomic<unsigned long long> hist[9];++ static std::atomic<int> seen{0};++ calls.fetch_add(1); jobsum.fetch_add((unsigned long long) n);++ hist[n < 8 ? n : 8].fetch_add(1);++ if (seen.fetch_add(1) % 256 == 255)++ std::fprintf(stderr, "mvlog calls=%llu jobs=%llu avg=%.2f hist1..4=%llu,%llu,%llu,%llu 5+=%llu,%llu,%llu,%llu\n",++ (unsigned long long) calls.load(), (unsigned long long) jobsum.load(),++ (double) jobsum.load() / (double) calls.load(),++ (unsigned long long) hist[1].load(), (unsigned long long) hist[2].load(),++ (unsigned long long) hist[3].load(), (unsigned long long) hist[4].load(),++ (unsigned long long) hist[5].load(), (unsigned long long) hist[6].load(),++ (unsigned long long) hist[7].load(), (unsigned long long) hist[8].load());++ }+ #endifBfJobs J[3] = {}; // the BF16 jobs, the MXFP4 ones, the Q8_0 ones: a launch eachint nj[3] = {0, 0, 0}, max_n[3] = {0, 0, 0}, max_c[3] = {1, 1, 1};bool vec[3] = {true, true, true};@@@ -1202,8 -1206,37 +1261,35 @@@vec[f] = vec[f] && xa && (f != 0 || (m.k % 8 == 0 && b.ld % 8 == 0 && ((uintptr_t) b.w & 15u) == 0));}}-#if defined(STRATA_METAL)- if (nq8 > 0) ::strata::dsv41::metal_host::q8_quant_multi(q8_batch, nq8);-#endif+ quant_flush(QI, s);for (int f = 0; f < 3; ++f) mv_set(f, J[f], nj[f], max_n[f], max_c[f], vec[f], s);+ #if defined(STRATA_METAL)+ if (std::getenv("DS4_DBG_Q8") && nj[2] > 0) { // DBG: the Q8_0 matvec's first job against a host recompute+ ds4_metal_flush();+ const BfJob& b = J[2].j[0];+ const uint8_t* w = (const uint8_t*) strata::gpu::host_view(b.w);+ const int8_t* xq = (const int8_t*) strata::gpu::host_view(b.xq);+ const float* xd = (const float*) strata::gpu::host_view(b.xd);+ const float* y = (const float*) strata::gpu::host_view(b.y);+ const float* xf = (const float*) strata::gpu::host_view(b.x);+ for (int c = 0; c < std::min(b.ncol, 2); ++c)+ for (int r = 0; r < 2; ++r) {+ double acc = 0.0, accf = 0.0;+ for (int kb = 0; kb < b.k / 32; ++kb) {+ const uint8_t* p = w + (size_t) r * b.ld + (size_t) kb * 34;+ uint16_t dh;+ std::memcpy(&dh, p, 2);+ float dw = __half2float(*(const __half*) &dh);+ int sum = 0;+ for (int j = 0; j < 32; ++j) sum += (int) (int8_t) p[2 + j] * (int) xq[(size_t) c * b.k + kb * 32 + j];+ acc += (double) dw * xd[(size_t) c * (b.k / 32) + kb] * sum;+ for (int j = 0; j < 32; ++j) accf += (double) dw * (int) (int8_t) p[2 + j] * xf[(size_t) c * b.xcs + kb * 32 + j];+ }+ std::fprintf(stderr, "DBG q8 job0 k=%d n=%d ncol=%d ld=%zu xcs=%zu c=%d r=%d recompute=%.5g exactf=%.5g kernel_y=%.5g xd0=%g xq0=%d\n", b.k, b.n, b.ncol,+ b.ld, b.xcs, c, r, acc, accf, y[(size_t) c * b.ycs + r], xd[(size_t) c * (b.k / 32)], (int) xq[(size_t) c * b.k]);+ }+ }+ #endifif (nij > 0 && !iq::mv(IJ, nij, s)) {fail("mv: a low-bit job not covered");return false;@@@ -1402,7 -1504,8 +1557,9 @@@ iq::SetArgs iq_set(const x3::SetArgs& sq.n_embd = sa.n_embd;q.n_ff = sa.n_ff;q.limit = sa.limit;+ q.xf16 = sa.x16;+ q.hbounds = sa.hbounds;+ q.total_rows = sa.total_rows;return q;}} // namespacediff --cc src/dsv41/ds4_gguf.cuhindex 05bb2696e,05bb2696e..4000093de--- a/src/dsv41/ds4_gguf.cuh+++ b/src/dsv41/ds4_gguf.cuh@@@ -52,6 -52,6 +52,10 @@@ const char* type_name(int type)/// Whether the device kernels take this type (MXFP4 is its own path; the i-quants and K-quants ds4_iq.cu instantiates)./// A false here with a layout known to type_row_bytes() means the CPU lane reads it and it must not enter the VRAM tier.bool type_on_device(int type);++/// Whether the *chunk* path's expert tile GEMM (ds4_mm.metal on Metal) takes this type. IQ1_M has a matvec kernel on++/// Metal but no tile kernel there, so a prompt in chunks would stream it to the GPU and stop; false here means the++/// prompts must run in windows instead (ds4_model.cu switches, and says so).++bool type_on_chunk(int type);/// The per-stream scratch of the MXFP4 experts on the current device (once per stream, as x3::prepare).bool prepare(cudaStream_t s);diff --cc src/dsv41/ds4_iq.cuindex da5e8e5d7,9d4965c44..186aedb9d--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@@ -2,9 -2,11 +2,13 @@@// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh). The block layouts and grids are// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "ds4_iq.cuh"+ #if defined(STRATA_METAL)+ #include <ds4_metal_host.h>+ #endif#include "ds4_numerics.cuh"+#include "ds4_experts_rule.hpp"#include <cuda_fp16.h>#if !defined(STRATA_USE_HIP)@@@ -1116,74 -954,29 +1120,74 @@@ bool tc_ok() { // as ds4_gguf.cu's: sdefault: return false; \}+// CUDA: a persistent grid (the resident blocks of the device, split over the groups), each block looping over row tiles with+// its codebook loaded once; HIP keeps the full grid (one tile a block)+template <auto Kernel>+int resident_ctas() { // (a static a kernel: the occupancy differs by the codebook's shared memory)- #if defined(STRATA_USE_HIP)++#if defined(STRATA_USE_HIP) || defined(STRATA_METAL)+ return 1 << 30;+#else+ static std::atomic<int> cache[16];+ int dev = 0;+ cudaGetDevice(&dev);+ if (dev < 0 || dev >= 16) return 1 << 30;+ int v = cache[dev].load(std::memory_order_relaxed);+ if (v == 0) {+ int sms = 0, per = 0;+ cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);+ cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per, Kernel, kThreads, 0);+ if (const char* e = std::getenv("DS4_IQ_MAX_SMS")) sms = std::min(sms, std::max(1, std::atoi(e))); // (benchmarking: a smaller card)+ if (const char* e = std::getenv("DS4_IQ_CTA_PER_SM")) per = std::min(per, std::max(1, std::atoi(e)));+ v = std::max(1, sms * std::max(per, 1));+ cache[dev].store(v, std::memory_order_relaxed);+ }+ return v;+#endif+}+// The grid: CUDA takes the device's resident blocks (at most one a work item) and every block loops over the items (group, row+// tile) round the grid (group by group, so that the blocks work on one expert's rows together): whichever groups are active,+// they share all the blocks, and a block loads its codebook once. HIP: a block an item (the full grid).+template <int T>+inline int items_grid(int items, int resident) { return std::max(1, std::min(items, resident)); }template <int T, int NE>-void launch_gu(const Args& a, dim3 grid, cudaStream_t s) { strata_ds4_launch(gate_up_kernel<T, NE>, grid, kThreads, 0, s, a); }+void launch_gu(const Args& a, int groups, cudaStream_t s) {+ const int tiles = a.n_ff / (kWarps * kGuRW);+ if constexpr (kPersist<T>)- gate_up_kernel<T, NE><<<items_grid<T>(groups * tiles, resident_ctas<gate_up_kernel<T, NE, kGuRW>>()), kThreads, 0, s>>>(a, groups);++ strata_ds4_launch(gate_up_kernel<T, NE>, items_grid<T>(groups * tiles, resident_ctas<gate_up_kernel<T, NE, kGuRW>>()), kThreads, 0, s, a, groups);+ else- gate_up_kernel<T, NE><<<dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s>>>(a, groups);++ strata_ds4_launch(gate_up_kernel<T, NE>, dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s, a, groups);+}template <int T, int NE>-void launch_dn(const Args& a, dim3 grid, cudaStream_t s) { strata_ds4_launch(down_kernel<T, NE>, grid, kThreads, 0, s, a); }+void launch_dn(const Args& a, int groups, cudaStream_t s) {+ const int tiles = a.n_embd / (kWarps * kDnRW);+ if constexpr (kPersist<T>)- down_kernel<T, NE><<<items_grid<T>(groups * tiles, resident_ctas<down_kernel<T, NE, kDnRW>>()), kThreads, 0, s>>>(a, groups);++ strata_ds4_launch(down_kernel<T, NE>, items_grid<T>(groups * tiles, resident_ctas<down_kernel<T, NE, kDnRW>>()), kThreads, 0, s, a, groups);+ else- down_kernel<T, NE><<<dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s>>>(a, groups);++ strata_ds4_launch(down_kernel<T, NE>, dim3((unsigned) tiles, (unsigned) groups), kThreads, 0, s, a, groups);+}template <int T>-void gu_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {- if (ne <= 1) launch_gu<T, 1>(a, grid, s);- else if (ne <= 4) launch_gu<T, 4>(a, grid, s);- else if (ne <= 8) launch_gu<T, 8>(a, grid, s);- else launch_gu<T, 16>(a, grid, s);+void gu_ne(const Args& a, int groups, int ne, cudaStream_t s) {+ if (ne <= 1) launch_gu<T, 1>(a, groups, s);+ else if (ne <= 4) launch_gu<T, 4>(a, groups, s);+ else if (ne <= 8) launch_gu<T, 8>(a, groups, s);+ else launch_gu<T, 16>(a, groups, s);}template <int T>-void dn_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {- if (ne <= 1) launch_dn<T, 1>(a, grid, s);- else if (ne <= 4) launch_dn<T, 4>(a, grid, s);- else if (ne <= 8) launch_dn<T, 8>(a, grid, s);- else launch_dn<T, 16>(a, grid, s);+void dn_ne(const Args& a, int groups, int ne, cudaStream_t s) {+ if (ne <= 1) launch_dn<T, 1>(a, groups, s);+ else if (ne <= 4) launch_dn<T, 4>(a, groups, s);+ else if (ne <= 8) launch_dn<T, 8>(a, groups, s);+ else launch_dn<T, 16>(a, groups, s);}+#include "ds4_iq_hip.inc"template <int T, bool GU>void launch_set(const SetArgs& a, dim3 grid, cudaStream_t s) {+#if defined(STRATA_USE_HIP)+ if (hipk::launch<T, GU>(a, s)) return; // (gfx12, STRATA_DS4_HIPK=1: the matrix-core tiles)+#endif#if !defined(STRATA_USE_HIP)if (tc_ok()) {- set_tc_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);+ strata_ds4_launch(set_tc_kernel<T, GU>, grid, kThreads, 0, s, a);return;}#endif@@@ -1229,17 -1020,10 +1233,17 @@@ bool light_on() { // STRATA_DS4_NO_LI}template <int T, bool GU>void launch_light(const SetArgs& a, const LightBuf& b, cudaStream_t s) {- const dim3 grid((unsigned) (((GU ? a.n_ff : a.n_embd) + kWarps - 1) / kWarps), (unsigned) a.n_exp);- if (a.max_rows <= 4) strata_ds4_launch(light_kernel<T, 4, GU>, grid, kThreads, 0, s, a, b.q, b.d);- else if (a.max_rows <= 8) strata_ds4_launch(light_kernel<T, 8, GU>, grid, kThreads, 0, s, a, b.q, b.d);- else strata_ds4_launch(light_kernel<T, kLightRows, GU>, grid, kThreads, 0, s, a, b.q, b.d);+ const int tiles = ((GU ? a.n_ff : a.n_embd) + kWarps - 1) / kWarps;+ if (a.max_rows <= 4) { // few rows: work items round a persistent grid (the codebook per block is then the cost)+ if constexpr (kPersist<T>)- light_kernel<T, 4, GU><<<items_grid<T>(a.n_exp * tiles, resident_ctas<light_kernel<T, 4, GU>>()), kThreads, 0, s>>>(a, b.q, b.d);++ strata_ds4_launch(light_kernel<T, 4, GU>, items_grid<T>(a.n_exp * tiles, resident_ctas<light_kernel<T, 4, GU>>()), kThreads, 0, s, a, b.q, b.d);+ else // (a small codebook: the release's kernel, which is the faster there)- light_kernel_rows<T, 4, GU><<<dim3((unsigned) tiles, (unsigned) a.n_exp), kThreads, 0, s>>>(a, b.q, b.d);++ strata_ds4_launch(light_kernel_rows<T, 4, GU>, dim3((unsigned) tiles, (unsigned) a.n_exp), kThreads, 0, s, a, b.q, b.d);+ } else { // more rows: compute bound, the release's kernel and grid+ const dim3 grid((unsigned) tiles, (unsigned) a.n_exp);- if (a.max_rows <= 8) light_kernel_rows<T, 8, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);- else light_kernel_rows<T, kLightRows, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);++ if (a.max_rows <= 8) strata_ds4_launch(light_kernel_rows<T, 8, GU>, grid, kThreads, 0, s, a, b.q, b.d);++ else strata_ds4_launch(light_kernel_rows<T, kLightRows, GU>, grid, kThreads, 0, s, a, b.q, b.d);+ }}bool light_set(const SetArgs& a, bool gu, cudaStream_t s) {const int K = gu ? a.n_embd : a.n_ff;@@@ -1419,13 -1094,7 +1423,13 @@@ static bool dq_use_old_kernel()bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {if (raw == nullptr || k % 256 != 0 || n <= 0) return false;const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);-#define DS4_DQ(T) strata_ds4_launch(dq_rows_kernel<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16)+ const bool old = dq_use_old_kernel();+#define DS4_DQ(T) do { \+ if (old) \- dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \++ strata_ds4_launch(dq_rows_kernel<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16); \+ else \- dq_rows_kernel2<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16); \++ strata_ds4_launch(dq_rows_kernel2<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16); \+ } while (0)switch (type) {case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;case kIQ2_XS: DS4_DQ(kIQ2_XS); break;@@@ -1447,6 -1116,6 +1451,12 @@@}bool supported(int t) {++#if defined(STRATA_METAL)++ // The MSL kernels (metal/src/engine/msl/ds4_iq.metal) implement the 2.0-bit types plus IQ1_M; IQ1_S and IQ4_NL++ // have CUDA kernels here but no Metal ones. Saying so makes those layers "cpu only" (ds4_model.cu, expert_layout),++ // so their experts stay on the CPU lane instead of stopping in "kernel launch with no Metal dispatch".++ if (t == kIQ1_S || t == kIQ4_NL) return false;++#endif// the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)switch (t) {case kIQ2_XXS: case kIQ2_XS: case kIQ2_S: case kIQ3_XXS: case kIQ3_S: case kIQ4_XS:@@@ -1513,18 -1182,32 +1523,30 @@@ const char* name(int t)}bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {+ #if defined(STRATA_METAL)+ // gate_up_kernel on the Metal dispatch (the prompt / window path's expert GEMM); the launch below is the fallback+ if (::strata::dsv41::metal_host::iq_gu(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,+ a.n_ff, a.limit, a.xq, a.xd, a.h, a.tg, groups))+ return true;+ #endifif (is_tq(a.tg)) return tq::gate_up(a, groups, ne_max, s);if (a.n_embd % 256 != 0 || a.n_ff % (kWarps * kGuRW) != 0) return false;- const dim3 grid((unsigned) (a.n_ff / (kWarps * kGuRW)), (unsigned) groups);-#define GU_(T) gu_ne<T>(a, grid, ne_max, s)+#define GU_(T) gu_ne<T>(a, groups, ne_max, s)DS4_IQ_SWITCH(a.tg, GU_)#undef GU_return true;}bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {+ #if defined(STRATA_METAL)+ // down_kernel on the Metal dispatch; the launch below is the fallback+ if (::strata::dsv41::metal_host::iq_dn(a.plan_ptr, a.grp_ptr, a.grp_start, a.grp_ent, a.n_groups, a.tr, a.n_embd,+ a.n_ff, a.hq, a.hd, a.part, a.td, groups))+ return true;+ #endifif (is_tq(a.td)) return tq::down(a, groups, ne_max, s);if (a.n_ff % 256 != 0 || a.n_embd % (kWarps * kDnRW) != 0) return false;- const dim3 grid((unsigned) (a.n_embd / (kWarps * kDnRW)), (unsigned) groups);-#define DN_(T) dn_ne<T>(a, grid, ne_max, s)+#define DN_(T) dn_ne<T>(a, groups, ne_max, s)DS4_IQ_SWITCH(a.td, DN_)#undef DN_return true;@@@ -1541,24 -1224,11 +1563,28 @@@ bool light_down(const SetArgs& a, cudaSreturn light_set(a, false, s);}+bool hipk_force(int mode) {+#if defined(STRATA_USE_HIP)+ hipk::force_mode().store(mode, std::memory_order_relaxed);+ return mode < 0 || hipk::ok_device() || mode == 0;+#else+ (void) mode;+ return false;+#endif+}+long long hipk_launches() {+#if defined(STRATA_USE_HIP)+ return hipk::launches().load(std::memory_order_relaxed);+#else+ return -1;+#endif+}+bool set_gate_up(const SetArgs& a, cudaStream_t s) {+ #if defined(STRATA_METAL)+ if (a.hbounds != nullptr && ::strata::dsv41::metal_host::iq_set_gu(a.wbase, a.stride, a.tr, a.hbounds, a.n_exp, a.total_rows, a.n_embd, a.n_ff, a.limit, a.xf, a.ldx, a.idx, a.h, a.ldo, a.tg, a.td))+ return true;+ #endifif (is_tq(a.tg)) return tq::set_gate_up(a, s);if (a.n_embd % 256 != 0 || a.n_ff % BN != 0) return false;SetArgs t = a;diff --cc src/dsv41/ds4_kernels.cuindex baf63dccd,f7edf5745..f6fdaccc8--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@@ -2,7 -2,10 +2,11 @@@// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.// src/ds4/ds4_kernels.cu - DeepSeek-V4.1's device kernels (see ds4_kernels.cuh). No fast math here: the rotary// angles, the softmaxes and the roundings follow the reference's F32 arithmetic. A window's token is blockIdx.y.++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "ds4_kernels.cuh"+ #if defined(STRATA_METAL)+ #include <ds4_metal_host.h>+ #endif#include "ds4_numerics.cuh"#include "ds4_q6k.cuh"#include "ds4_attn_split.cuh"@@@ -2149,21 -1929,24 +2197,43 @@@ void finish(float* x, int n_vec, int le}void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {- hc_mix_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes);+ #if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::hc_mix(h, fn, hcdim, n_mix, T, eps, mixes)) return;+ #endif+ strata_ds4_launch(hc_mix_kernel, dim3(n_mix, T), kThreads, 0, s, h, fn, hcdim, eps, mixes);}+void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, unsigned* cnt,+ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,+ float* comb, cudaStream_t s) {- hc_mix_split_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes, cnt, scale, base, hc, iters, hc_eps, pre,++ strata_ds4_launch(hc_mix_split_kernel, dim3(n_mix, T), kThreads, 0, s, h, fn, hcdim, eps, mixes, cnt, scale, base, hc, iters, hc_eps, pre,+ post, comb);+}void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,float* pre, float* post, float* comb, cudaStream_t s) {- hc_split_kernel<<<T, 32, 0, s>>>(mixes, scale, base, hc, iters, hc_eps, pre, post, comb);+ #if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::hc_split(mixes, scale, base, hc, iters, hc_eps, T, pre, post, comb)) return;+ #endif+ strata_ds4_launch(hc_split_kernel, T, 32, 0, s, mixes, scale, base, hc, iters, hc_eps, pre, post, comb);+ }++void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes,++ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,++ float* comb, cudaStream_t s) {++#if defined(STRATA_METAL)++ // one dispatch: the mix and the sinkhorn split of its result (bit-identical to the two, see ds4_init.metal)++ if (strata::dsv41::metal_host::hc_mix_split(h, fn, hcdim, n_mix, T, eps, mixes, scale, base, hc, iters, hc_eps,++ pre, post, comb))++ return;++#endif++ hc_mix(h, fn, hcdim, n_mix, T, eps, mixes, s);++ hc_split(mixes, scale, base, hc, iters, hc_eps, T, pre, post, comb, s);+}++void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,cudaStream_t s) {- hc_pre_norm_kernel<<<T, 1024, 0, s>>>(h, mix, w, dim, hc, eps, x);+ #if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::hc_pre_norm(h, mix, w, dim, hc, T, eps, x)) return;+ #endif+ strata_ds4_launch(hc_pre_norm_kernel, T, 1024, 0, s, h, mix, w, dim, hc, eps, x);}void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,float* out, cudaStream_t s) {@@@ -2318,17 -2164,9 +2451,17 @@@ void sparse_attn(const float* q, constheads, d, scale, out);return;}+ // DS4_FAST_ATTN=1: the chunked prompt's matrix-core path (16 heads a block, a 16-key tile). The decode and the+ // verify windows keep the scalar kernels: one query's heads are not the matrix's rows there.+#if defined(STRATA_USE_HIP)+ if (fast_attn_enabled() && a.gen != 0 && T >= kFaKeys && heads % kFaHeads == 0) {- sparse_attn_wmma_kernel<<<dim3(heads / kFaHeads, T), kThreads, 0, s>>>(q, win, (const uint8_t*) ckv, sel, sel_stride,++ strata_ds4_launch(sparse_attn_wmma_kernel, dim3(heads / kFaHeads, T), kThreads, 0, s, q, win, (const uint8_t*) ckv, sel, sel_stride,+ a, sink, heads, scale, out);+ return;+ }+#endif// packed rows: as many heads a block as still leaves two blocks a multiprocessor (a decoded row serves them all)- int G = heads_per_block;- if (G != 1 && G != 2 && G != 4 && G != 8) {+ int G = heads_per_block; if (G != 1 && G != 2 && G != 4 && G != 8) {G = 8;while (G > 1 && (heads % G != 0 || (heads / G) * T < 2 * sm_count())) G /= 2;}@@@ -2361,20 -2221,34 +2516,37 @@@ void ring_restore_batch(const RingBatchkv_row_bytes(kKvFp8, d));}void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s) {- hc_mean_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(h, hc, dim, out, out_stride);+ #if defined(STRATA_METAL)+ if (std::getenv("DS4_SKIP_HCMEAN")) return;+ #endif+ #if defined(STRATA_METAL)+ // hc_mean_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::hc_mean(h, T, hc, dim, out, out_stride)) return;+ #endif+ strata_ds4_launch(hc_mean_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, h, hc, dim, out, out_stride);+ }+ void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) {+ #if defined(STRATA_METAL)+ // iota_rows_kernel on the Metal dispatch; the launch below is the fallback+ if (strata::dsv41::metal_host::iota_rows(out, T, n, stride)) return;+ #endif+ strata_ds4_launch(iota_rows_kernel, T, 64, 0, s, out, n, stride);}- void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { iota_rows_kernel<<<T, 64, 0, s>>>(out, n, stride); }void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {- markov_kernel<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, H, tok, V, R);+ #if defined(STRATA_METAL)+ // markov_kernel on the Metal dispatch (the drafter's Markov bias); the launch below is the fallback+ if (strata::dsv41::metal_host::markov_add(logits, E, H, tok, V, R)) return;+ #endif+ strata_ds4_launch(markov_kernel, cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, E, H, tok, V, R);}+void markov_add_q8(float* logits, const uint16_t* E, const uint8_t* Hq, const int* tok, int V, int R, cudaStream_t s) {- markov_kernel_q8<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, Hq, tok, V, R);++ strata_ds4_launch(markov_kernel_q8, cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, E, Hq, tok, V, R);+}void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s) {- markov_share_kernel<<<cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, H, erow, vn, R);+ strata_ds4_launch(markov_share_kernel, cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, H, erow, vn, R);}void argmax_share(const float* row, int vn, int v0, unsigned* slot, unsigned seq, cudaStream_t s) {- argmax_share_kernel<<<1, 1024, 0, s>>>(row, vn, v0, slot, seq);+ strata_ds4_launch(argmax_share_kernel, 1, 1024, 0, s, row, vn, v0, slot, seq);}void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E, uint16_t* erow, int R,cudaStream_t s) {@@@ -2406,38 -2300,18 +2598,38 @@@ void xfer_put(const Xfer& x, unsigned*size_t total = 0;for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));- xfer_put_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq, cnt);+ strata_ds4_launch(xfer_put_kernel, blocks, kThreads, 0, s, x, flag, seq, cnt);+ // (another GPU, or the host, waits for this flag: on Windows the put must not sit in the queue - #303: two RTX 3080s+ // hung at the first hand-off, the receiver spinning on a flag whose writer had never been submitted)+ win_submit(s);+}+void win_submit(cudaStream_t s) {+#if defined(_WIN32)+ static const bool on = [] {+ const char* v = std::getenv("DS4_WIN_SUBMIT");+ return v == nullptr || std::atoi(v) != 0;+ }();+ if (on) {+ (void) cudaStreamQuery(s);+ (void) cudaGetLastError();+ }+#else+ (void) s;+#endif}void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s) {size_t total = 0;for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));- xfer_get_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq);+ strata_ds4_launch(xfer_get_kernel, blocks, kThreads, 0, s, x, flag, seq);}void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s) {- if (n > 0) wait_flags_kernel<<<1, 32, 0, s>>>(flags, n, stride, seq);+ if (n > 0) strata_ds4_launch(wait_flags_kernel, 1, 32, 0, s, flags, n, stride, seq);}-void set_flag(unsigned* flag, unsigned v, cudaStream_t s) { strata_ds4_launch(set_flag_kernel, 1, 1, 0, s, flag, v); }+void set_flag(unsigned* flag, unsigned v, cudaStream_t s) {- set_flag_kernel<<<1, 1, 0, s>>>(flag, v);++ strata_ds4_launch(set_flag_kernel, 1, 1, 0, s, flag, v);+ win_submit(s); // (the writer on another GPU waits for this acknowledgement before its next put - #384)+}void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s) {const int slot = (int) (seq % (unsigned) a.slots);if (a.nm > 2 && a.sums != nullptr && a.bflags != nullptr) { // (the reduce-scatter: ar_rs_kernel)diff --cc src/dsv41/ds4_kernels.cuhindex 08075fa00,72488e9b3..2727dc6df--- a/src/dsv41/ds4_kernels.cuh+++ b/src/dsv41/ds4_kernels.cuh@@@ -61,13 -61,8 +61,17 @@@ void finish(float* x, int n_vec, int le/// pre / post / comb as hc_split_sinkhorn (comb[t][j * hc + k], j the source stream, k the destination). Strides: mixes/// 32, pre / post 8, comb 64 floats a row.void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s);+/// hc_mix then hc_split in one launch (cnt: T zeroed unsigned counters, left zero; n_mix <= 32, T <= kWin): the last block of a token+/// runs the split - the same arithmetic as the two launches+void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, unsigned* cnt,+ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre, float* post,+ float* comb, cudaStream_t s);void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,float* pre, float* post, float* comb, cudaStream_t s);++/// the same two in one dispatch (the Metal kernel ds4_hc_mix_split); the CUDA path calls hc_mix then hc_split.++void hc_mix_split(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes,++ const float* scale, const float* base, int hc, int iters, float hc_eps, float* pre,++ float* post, float* comb, cudaStream_t s);/// x[t][e] = sum_s mix[t][s] h[t][s][e], then x = rmsnorm(x) * w (w null: no norm).void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,cudaStream_t s);diff --cc src/dsv41/ds4_launch.hindex 000000000,7d0bc9ad8..433f833aemode 000000,100644..100644--- a/src/dsv41/ds4_launch.h+++ b/src/dsv41/ds4_launch.h@@@ -1,0 -1,28 +1,28 @@@+ // strata_ds4_launch(kernel, grid, block, shared, stream, args...): the engine's one spelling of a kernel launch. On CUDA it is+ // kernel<<<grid, block, shared, stream>>>(args...); on Metal metal/include/metal_compat/cuda_launch.h maps the same call onto-// a dispatch. (Force-included for CUDA compiles by CMakeLists.txt.)++// a dispatch. (Every file that launches kernels includes this header.)+ #pragma once-#if defined(__CUDACC__) && !defined(STRATA_METAL)++#if (defined(__CUDACC__) || defined(__HIPCC__) || defined(__HIP__)) && !defined(STRATA_METAL)+ #include <cuda_runtime.h>+ #include <cstdio>+ #include <cstdlib>+ template <class... KA, class... CA>+ inline void strata_ds4_launch_at(const char* file, int line, void (*kernel)(KA...), dim3 grid, dim3 block, size_t shared,+ cudaStream_t stream, CA&&... args) {+ kernel<<<grid, block, shared, stream>>>(static_cast<KA>(args)...);+ static const bool check = std::getenv("DS4_LAUNCH_CHECK") != nullptr; // report the launch that failed (not a later call)+ if (check) {+ const cudaError_t e = cudaGetLastError();+ if (e != cudaSuccess)+ std::fprintf(stderr, "ds4 launch failed at %s:%d: %s (grid %u,%u,%u block %u,%u,%u shared %zu)\n", file, line,+ cudaGetErrorString(e), grid.x, grid.y, grid.z, block.x, block.y, block.z, shared);+ }+ }+ #define strata_ds4_launch(...) strata_ds4_launch_at(__FILE__, __LINE__, __VA_ARGS__)++ // (the Metal debug dumps in ds4_model.cu name these; on CUDA there is no deferred queue and a device pointer is not a host view)+ inline void ds4_metal_flush() {}+ namespace strata::gpu {+ inline void* host_view(const void* p) { return const_cast<void*>(p); }+ }+ #endifdiff --cc src/dsv41/ds4_main.cppindex 3b259b2af,4916954c7..d576e348f--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@@ -11,18 -11,14 +11,24 @@@// <decode ms> <stop|length|cancel> 0 0 <reused> 0 0`; `STOP` ends a request early, `QUIT` the engine. A prompt that// continues the state (the conversation so far) reuses it; one that only shares the last prompt's start goes back to// that prompt's end; anything else starts over. `GENI <max_new> [keys] <file> <ids>` carries the prompt's pictures-// (the file: see read_pics); the state is shared only where the same picture sits at the same place.+// (the file: see read_pics); the state is shared only where the same picture sits at the same place. With+// STRATA_DS4_SLOT_KEEP=1 the conversations are kept when another one takes the model (a slot each: the state, the+// compressed KV included), and across runs in STRATA_DS4_SLOT_DIR - so a request that does not continue the state but+// does continue a kept conversation reads only its new tokens.+// `pin=N` on a GEN marks the first N prompt tokens as a start other requests share (a system prompt and its tools): the+// read stops there for a checkpoint, so a request behind the same start - another agent - goes back to it instead of+// reading it again (also without slots). STRATA_DS4_REUSE=0 turns every reuse off (each request reads its whole prompt).+ #if defined(__APPLE__)+ #include <execinfo.h>+ #include <csignal>+ #include <unistd.h>++#include <libproc.h> // proc_pid_rusage: the bytes this process had the storage read (io_read_bytes)+ #endif#include "ds4_model.hpp"+#include "ds4_experts_rule.hpp"+#include "ds4_os.hpp"#include "ds4_packinfo.hpp"+#include "ds4_slotmeta.hpp"#include <algorithm>#include <cmath>@@@ -52,8 -42,8 +58,8 @@@ double now_ms()return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();}--// the bytes this process had the storage read for it (Linux's /proc/self/io; -1 where there is none): the Monitor's--// disk reads a token, the experts read in place from the model's files included++// the bytes this process had the storage read for it (Linux's /proc/self/io, macOS's rusage; -1 where there is none): the++// Monitor's disk reads a token, the experts read in place from the model's files includedlong long io_read_bytes() {#ifdef __linux__std::FILE* f = std::fopen("/proc/self/io", "r");@@@ -64,6 -54,6 +70,14 @@@if (std::sscanf(line, "read_bytes: %lld", &v) == 1) break;std::fclose(f);return v;++#elif defined(__APPLE__)++ // (the same number `iotop`/Activity Monitor's "Bytes read" shows: every read the storage did for this process, the++ // page cache's misses of the experts read in place included; a page-cache hit counts no storage read, which is what++ // the 60 tps go/no-go needs)++ struct rusage_info_v4 ri;++ std::memset(&ri, 0, sizeof ri);++ if (proc_pid_rusage(getpid(), RUSAGE_INFO_V4, (rusage_info_t*) &ri) != 0) return -1;++ return (long long) ri.ri_diskio_bytesread;#elsereturn -1;#endif@@@ -86,8 -76,8 +100,8 @@@ std::vector<int> parse_ids(const std::svoid usage() {std::fprintf(stderr,"maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"-- " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"- " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"++ " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--profile-readonly] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"+ " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none|auto] [--dspark-block N] [--dspark-conf F] [--experts N|auto]\n"" [--vision on|off] [--mmap-experts on|off|auto]\n"" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"" | --encode-image FILE.rgb --size WxH --dump-image OUT\n"@@@ -166,9 -144,42 +181,42 @@@ float dspark_cum_default(const strata::// One round of decoding after `tok` (the token at pos(), not fed yet): with the drafter, its drafts while their// confidence holds (at most `room` - 1 of them), verified together; else a plain step. `out` gets the tokens that follow// tok (the accepted drafts, then the trunk's next), the last of them the next round's tok.+ struct RoundTimes { // DS4_ROUND_TIMES: where a decode round's time goes (ms, summed), printed at exit+ double draft = 0, verify = 0, step = 0;+ unsigned long long draft_d = 0, verify_d = 0, step_d = 0;+ long rounds = 0, spec_rounds = 0, steps = 0;+ ~RoundTimes() {+ if (std::getenv("DS4_ROUND_TIMES") && rounds > 0)+ std::fprintf(stderr, "rounds: %ld (%ld speculative, %ld plain steps): draft %.1f ms (%.2f a round), verify %.1f ms (%.2f), plain step %.1f ms (%.2f)\n",+ rounds, spec_rounds, steps, draft, spec_rounds ? draft / (double) spec_rounds : 0.0, verify, spec_rounds ? verify / (double) spec_rounds : 0.0,+ step, steps ? step / (double) steps : 0.0);+ if (std::getenv("DS4_ROUND_TIMES") && rounds > 0)+ std::fprintf(stderr, "dispatches a round: draft %.0f, verify %.0f, plain step %.0f\n", spec_rounds ? (double) draft_d / (double) spec_rounds : 0.0, spec_rounds ? (double) verify_d / (double) spec_rounds : 0.0, steps ? (double) step_d / (double) steps : 0.0);+ }+ } g_rt;+ #if defined(STRATA_METAL)+ extern "C" unsigned long long ds4_dispatch_seq();+ #endif+ #if defined(STRATA_METAL)+ extern "C" void ds4_timeline_gate(int);+ #endif+ static void tl_gate(bool open) {+ #if defined(STRATA_METAL)+ static const bool only_draft = std::getenv("DS4_TL_DRAFT") != nullptr;+ if (only_draft) ds4_timeline_gate(open ? 1 : 0);+ #endif+ }+ static unsigned long long rt_seq() {+ #if defined(STRATA_METAL)+ return ds4_dispatch_seq();+ #else+ return 0;+ #endif+ }+ static double rt_now() { return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count(); }bool round(strata::ds4::Model& m, int tok, int room, bool spec, float conf_min, float temperature, float top_p, int top_k,float min_p, unsigned long long seed, unsigned long long& counter, std::vector<int>& out, int& offered,- int& accepted, std::string& err) {+ int& accepted, std::string& err, float cum_min = -1.0f) {out.clear();offered = accepted = 0;if (spec && room > 1 && m.draft_block() > 0) {@@@ -178,11 -198,7 +235,11 @@@g_round_conf = conf;g_round_nd = (int) drafts.size();int keep = 0;- while (keep < (int) drafts.size() && keep < room - 1 && keep < g_spec_depth && conf[(size_t) keep] >= conf_min) ++keep;+ if (cum_min < 0) cum_min = dspark_cum_default(m);+ float cum = 1.0f;- while (keep < (int) drafts.size() && keep < room - 1 && conf[(size_t) keep] >= conf_min &&++ while (keep < (int) drafts.size() && keep < room - 1 && keep < g_spec_depth && conf[(size_t) keep] >= conf_min &&+ (cum *= conf[(size_t) keep]) >= cum_min)+ ++keep;if (keep > 0) {drafts.resize((size_t) keep);int acc = 0, next = -1;@@@ -1182,15 -791,24 +1249,28 @@@ int spec_eval(strata::ds4::Model& m, co}} // namespace+#ifndef DS4_BUILD_ID+#define DS4_BUILD_ID "unknown"+#endif++ #if defined(STRATA_METAL)+ #include "sampler_stat.inc"+ #endifint main(int argc, char** argv) {+ strata::ds4::os::fine_timer();+ #if defined(STRATA_METAL)+ if (argc > 1 && std::string(argv[1]) == "--sampler-test") return sampler_stat::run(argc - 1, argv + 1); // (the sampler's statistics harness)+ #endif+ #if defined(__APPLE__)+ if (std::getenv("DS4_BT")) { // a crash prints where it was (no debugger on the Mac box)+ signal(SIGSEGV, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(139); });+ signal(SIGBUS, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(138); });+ signal(SIGABRT, [](int) { void* fr[40]; const int n = backtrace(fr, 40); backtrace_symbols_fd(fr, n, 2); _exit(134); });+ }+ #endif-strata::ds4::Options o;std::string ids_s, dump, eos = "1";- int n = 0, window = 16;+ int n = 0, window = 0; // (0: the model's feed unit - a whole prompt on the chunked path; 16 for the scoring runs)bool spec = true;float temp = 0.0f; // the CLI's sampling (0: greedy)unsigned long long cli_seed = 1;@@@ -1213,9 -831,9 +1293,10 @@@for (int g : parse_ids(val("--gpus"))) o.gpus.push_back(g);} else if (a == "--max-context") o.max_context = std::atoll(val("--max-context").c_str());else if (a == "--reserve-mib") o.reserve_mib = std::atoll(val("--reserve-mib").c_str());- else if (a == "--vram-experts") o.vram_experts = std::atof(val("--vram-experts").c_str());+ else if (a == "--vram-experts") { o.vram_experts = std::atof(val("--vram-experts").c_str()); o.vram_experts_auto = false; }else if (a == "--cpu-threads") o.cpu_threads = std::atoi(val("--cpu-threads").c_str());else if (a == "--expert-profile") o.profile = val("--expert-profile");++ else if (a == "--profile-readonly") o.profile_readonly = true; // (read the placement, never write the counts back)else if (a == "--ssd-cache") o.ssd_cache_gb = std::atof(val("--ssd-cache").c_str());else if (a == "--ssd-policy") {const std::string v = val("--ssd-policy");@@@ -1227,13 -845,8 +1308,14 @@@const std::string v = val("--dspark");spec = v != "off";o.dspark = v != "none"; // none: not even loaded+ o.dspark_auto = v == "auto";} else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());+ else if (a == "--experts") {+ const std::string v = val("--experts");+ o.experts = v == "auto" ? -1 : std::max(1, std::atoi(v.c_str()));+ }+ else if (a == "--dspark-block") o.dspark_block = std::max(0, std::atoi(val("--dspark-block").c_str()));+ else if (a == "--dspark-depth") g_spec_depth = std::max(1, std::atoi(val("--dspark-depth").c_str()));else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));else if (a == "--threads") o.threads = val("--threads") != "off";@@@ -1397,13 -992,16 +1482,21 @@@std::vector<int> got;for (;;) {out.push_back(tok);+ {+ const double tn = now_ms();+ tok_ms.push_back(tn - t_prev);+ t_prev = tn;+ }std::printf("%d,", tok);std::fflush(stdout);+ if (std::getenv("DS4_PROGRESS") && out.size() % 200 == 0) { // (sustained-run time series)+ static double tp = now_ms();+ static size_t np = 0;+ const double tn = now_ms();+ std::fprintf(stderr, "progress %zu tokens: %.2f tok/s over the last %zu\n", out.size(), 1000.0 * (double) (out.size() - np) / (tn - tp), out.size() - np);+ tp = tn;+ np = out.size();+ }if ((tok == 1 && std::getenv("DS4_NO_EOS") == nullptr) || (int) out.size() >= n) break; // (DS4_NO_EOS: decode n tokens whatever)if (got.empty()) {int off = 0, acc = 0;diff --cc src/dsv41/ds4_model.cuindex 91d29a266,4f605d077..c0ac558c8--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@@ -1,9744 -1,8826 +1,10417 @@@-// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).-// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.-// src/ds4/ds4_model.cu - DeepSeek-V4.1-Flash on Maya (see ds4_model.hpp).-//-// The layer, for one token at position p (DeepSeek's inference/model.py, Block.forward with the lagged pre-mix):-// [Engram, layers 1 and 14] h += gate(h, key) * value from 24 n-gram rows of the FP8 tables-// a_pre, a_post, a_comb = hc_mixes(h); x = rmsnorm(sum_s pre[s] h[s]) (pre: the previous layer's f_pre)-// attention(x): q from the low-rank wq_a / wq_b, one shared K = V row per token in a 128-slot window ring, plus-// (compress ratio > 0) up to 512 compressed rows the indexer picks; a learned sink per head; the-// output's rotary part rotated back, rounded to FP8, through the grouped wo_a and wo_b-// h = post * attn + comb^T h; f_* = hc_mixes(h); x = rmsnorm(sum_s a_pre[s] h[s])-// moe(x): 6 of 384 routed experts (sqrt-softplus scores, biased top-k) + the shared expert-// h = post * moe + comb^T h; pre = f_pre-// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).-#include "ds4_model.hpp"-#include "ds4_engram_io.hpp"--#include "ds4_gguf.cuh"-#include "ds4_gguf_cpu.hpp"-#include "ds4_kernels.cuh"-#include "ds4_os.hpp"-#include "ds4_pool.hpp"-#include "ds4_store.hpp"-#include "ds4_vision.cuh"-#include "strata/dsv41/exl3_cpu.hpp"-#include "strata/dsv41/exl3.hpp"-#include "strata/kernels/f16_bits.hpp"-#include "strata/dsv41/moe_win.hpp"-#include "strata/dsv41/sampler.hpp"--#include <cublas_v2.h>-#include <cuda_runtime.h>--#if defined(STRATA_METAL)-extern "C" void ds4_busy_begin();-extern "C" void ds4_busy_end();-#include <sys/sysctl.h>-#include <mach/mach.h>-#endif-#include <algorithm>-#include <array>-#include <condition_variable>-#include <deque>-#include <atomic>-#include <chrono>-#include <cmath>-#include <cstdio>-#include <cstdlib>-#include <cstring>-#include <fstream>-#include <functional>-#include <future>-#include <map>-#include <mutex>-#include <numeric>-#include <sstream>-#include <thread>-#include <unordered_map>--#if defined(_MSC_VER)-#define strtok_r strtok_s-#endif--namespace strata::ds4 {-namespace x3 = strata::dsv41k::exl3;-namespace kc = strata::dsv41k::cpu;-namespace gf = strata::dsv41k::glmf;-namespace gg = strata::ds4::gguf;--#define CK(call) \- do { \- cudaError_t e_ = (call); \- if (e_ != cudaSuccess) { \- err = std::string(#call) + ": " + cudaGetErrorString(e_); \- return false; \- } \- } while (0)--namespace {--double now_ms() {- return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();-}--size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }--// part p of np of `whole` (a multiple of 128) in 128-blocks, as even as they go (whole / 128 % np parts a block more:-// those from part rot on, cyclically - so a group's layers, each its own rot, even out what each member holds):-// tensor parallel's columns of an FFN or of the head - an even split is what whole / np gave before-void split128(int whole, int np, int p, int& off, int& n, int rot = 0) {- const int b = whole / 128, q = b / np, r = b % np;- auto blocks = [&](int i) { return q + (((i - rot) % np + np) % np < r ? 1 : 0); };- int at = 0;- for (int i = 0; i < p; ++i) at += blocks(i);- off = 128 * at;- n = 128 * blocks(p);-}--std::string read_file(const std::string& p) {- std::ifstream f(p);- std::stringstream ss;- ss << f.rdbuf();- return ss.str();-}-std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"- std::vector<int> v;- std::stringstream ss(s);- std::string part;- while (std::getline(ss, part, ',')) {- if (part.empty() || part == "\n") continue;- const size_t dash = part.find('-');- const int a = std::atoi(part.c_str());- const int b = dash == std::string::npos ? a : std::atoi(part.c_str() + dash + 1);- for (int c = a; c <= b; ++c) v.push_back(c);- }- return v;-}-// each NUMA node's physical cores (the first logical CPU of each core), in order-std::vector<std::vector<int>> node_cores() {-#if defined(_WIN32)- static const bool smt = [] { // DS4_CPU_SMT=1: the second hardware thread of every core as well (memory-bound rows gain from it)- const char* v = std::getenv("DS4_CPU_SMT");- return v != nullptr && std::atoi(v) != 0;- }();- std::vector<std::vector<int>> nodes = os::node_cores(smt);-#else- std::vector<std::vector<int>> nodes;- for (int n = 0; n < 64; ++n) {- const std::string cl = read_file("/sys/devices/system/node/node" + std::to_string(n) + "/cpulist");- if (cl.empty()) break;- std::vector<int> cores;- for (int c : cpu_list(cl)) {- const std::vector<int> sib = cpu_list(- read_file("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list"));- if (sib.empty() || sib[0] == c) cores.push_back(c);- }- if (!cores.empty()) nodes.push_back(cores);- }-#endif- if (nodes.empty()) {- std::vector<int> all;- for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency() / 2); ++c) all.push_back((int) c);- nodes.push_back(all);- }- return nodes;-}-// anonymous memory on one NUMA node (node < 0: wherever the first touch puts it), transparent huge pages-void* node_map(size_t bytes, int node, int n_nodes) { return os::anon_alloc(bytes, node, n_nodes); }-void pin_self(int cpu) { os::pin_self(cpu); }-// the RAM this process can still get: MemAvailable, and below it the tightest cgroup v2 limit on the way to the root-// (its limit less what is charged, the page cache counted as free: the kernel takes it back); ~0: not known-uint64_t ram_avail() {-#if defined(_WIN32)- return os::ram_avail();-#endif- uint64_t avail = ~uint64_t{0};- {- std::istringstream mi(read_file("/proc/meminfo"));- std::string k;- uint64_t kb = 0;- while (mi >> k >> kb) {- if (k == "MemAvailable:") avail = kb << 10;- mi.ignore(64, '\n');- }- }- std::string cg = read_file("/proc/self/cgroup"); // "0::/user.slice/..."- const size_t at = cg.find("0::");- if (at == std::string::npos) return avail;- cg = cg.substr(at + 3);- cg = cg.substr(0, cg.find('\n'));- for (;;) {- const std::string dir = "/sys/fs/cgroup" + (cg == "/" ? std::string() : cg);- const std::string mx = read_file(dir + "/memory.max");- if (!mx.empty() && mx.compare(0, 3, "max") != 0) {- const uint64_t lim = std::strtoull(mx.c_str(), nullptr, 10);- const uint64_t cur = std::strtoull(read_file(dir + "/memory.current").c_str(), nullptr, 10);- uint64_t file = 0;- std::istringstream st(read_file(dir + "/memory.stat"));- std::string k;- uint64_t v = 0;- while (st >> k >> v)- if (k == "file") file = v;- const uint64_t used = cur > file ? cur - file : 0;- avail = std::min(avail, lim > used ? lim - used : 0);- }- if (cg.empty() || cg == "/") break;- const size_t sl = cg.rfind('/');- cg = sl == 0 || sl == std::string::npos ? "/" : cg.substr(0, sl);- }- return avail;-}--float bf16_to_f32(uint16_t b) {- uint32_t u = (uint32_t) b << 16;- float f;- std::memcpy(&f, &u, 4);- return f;-}-float f16_to_f32(uint16_t h) {- const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;- uint32_t u;- if (e == 0) {- if (m == 0) u = s;- else { // subnormal- int ee = -1;- uint32_t mm = m;- do {- ++ee;- mm <<= 1;- } while ((mm & 0x400) == 0);- u = s | (uint32_t) (127 - 15 - ee) << 23 | (mm & 0x3ff) << 13;- }- } else if (e == 31) u = s | 0x7f800000u | m << 13;- else u = s | (e - 15 + 127) << 23 | m << 13;- float f;- std::memcpy(&f, &u, 4);- return f;-}--// ------------------------------------------------------------------ the pack-struct Tensor {- std::string dtype;- int file = -1;- uint64_t off = 0, bytes = 0;- std::vector<int64_t> shape;-};--struct Pack {- std::string dir, model_dir;- std::unordered_map<std::string, std::string> kv;- std::vector<std::string> files;- std::vector<os::fd_t> fds;- std::unordered_map<std::string, Tensor> t;- int eg_vocab = 0, eg_layers = 0, eg_ngram = 0, eg_heads = 0, eg_hd = 0, eg_pad = 0;- std::vector<int32_t> eg_map;- std::vector<int64_t> eg_mult, eg_primes, eg_off;- // format gguf (tools/ds4/pack_gguf.py): the files are a GGUF's shards, every one mapped at load (map()) - the- // routed experts are always read in place (the OS page cache their RAM tier, the disk the rest)- bool gguf = false;-- std::vector<std::pair<uint8_t*, size_t>> maps; // file -> its read-only mapping (map(): the experts in place)- std::vector<os::fd_t> dfds; // (gguf) file -> unbuffered (os::open_direct; kBadFd: buffered reads only)- std::vector<os::fd_t> pfds; // (gguf) file -> for reads from many threads at once (os::open_par)-- ~Pack() {- for (auto& m : maps) os::unmap(m.first, m.second);- for (os::fd_t fd : fds) os::close_fd(fd);- for (os::fd_t fd : dfds) os::close_fd(fd);- for (os::fd_t fd : pfds) os::close_fd(fd);- }- // file i mapped whole, read-only (once; not thread-safe: called while loading)- const uint8_t* map(int file) {- if (maps.size() < fds.size()) maps.resize(fds.size(), {nullptr, 0});- auto& m = maps[(size_t) file];- if (m.first == nullptr) {- const uint64_t size = os::file_size(fds[(size_t) file]);- void* p = os::map_ro(fds[(size_t) file], size);- if (p == nullptr) return nullptr;- m = {(uint8_t*) p, (size_t) size};- }- return m.first;- }-- bool load(const std::string& d, std::string& err) {- dir = d;- {- std::ifstream f(d + "/ds4.txt");- if (!f) {- err = d + "/ds4.txt: not found (run tools/ds4/pack.py --model <model>)";- return false;- }- std::string line;- while (std::getline(f, line)) {- const size_t sp = line.find(' ');- if (sp == std::string::npos) continue;- kv[line.substr(0, sp)] = line.substr(sp + 1);- }- }- model_dir = str("model_dir");- std::FILE* f = std::fopen((d + "/tensors.txt").c_str(), "r");- if (f == nullptr) {- err = d + "/tensors.txt: not found";- return false;- }- std::vector<char> buf(1 << 16);- while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {- char* s = buf.data();- if (s[0] == 'f') { // file <i> <name>- char name[4096];- int i = 0;- if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {- if ((int) files.size() <= i) files.resize((size_t) i + 1);- files[(size_t) i] = name;- }- continue;- }- if (s[0] != 't') continue;- char* save = nullptr;- ::strtok_r(s, " \n", &save);- const char* name = ::strtok_r(nullptr, " \n", &save);- const char* dt = ::strtok_r(nullptr, " \n", &save);- const char* fi = ::strtok_r(nullptr, " \n", &save);- const char* off = ::strtok_r(nullptr, " \n", &save);- const char* nb = ::strtok_r(nullptr, " \n", &save);- if (name == nullptr || nb == nullptr) continue;- Tensor x;- x.dtype = dt;- x.file = std::atoi(fi);- x.off = std::strtoull(off, nullptr, 10);- x.bytes = std::strtoull(nb, nullptr, 10);- while (const char* dim = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(dim));- t.emplace(name, std::move(x));- }- std::fclose(f);- for (const std::string& fn : files) {- const std::string p = model_dir + "/" + fn;- const os::fd_t fd = os::open_ro(p);- if (fd == os::kBadFd) {- err = p + ": cannot open";- return false;- }- fds.push_back(fd);- }- gguf = str("format") == "gguf";- if (gguf)- for (size_t i = 0; i < fds.size(); ++i) {- if (map((int) i) == nullptr) {- err = model_dir + "/" + files[i] + ": cannot map";- return false;- }- dfds.push_back(os::open_direct(model_dir + "/" + files[i]));- pfds.push_back(os::open_par(model_dir + "/" + files[i]));- }- // the Engram tables- std::FILE* e = std::fopen((d + "/engram.bin").c_str(), "rb");- if (e == nullptr) {- err = d + "/engram.bin: not found";- return false;- }- char magic[4];- int32_t hdr[6];- bool ok = std::fread(magic, 1, 4, e) == 4 && std::memcmp(magic, "DSE1", 4) == 0 &&- std::fread(hdr, 4, 6, e) == 6;- if (ok) {- eg_vocab = hdr[0], eg_layers = hdr[1], eg_ngram = hdr[2], eg_heads = hdr[3], eg_hd = hdr[4], eg_pad = hdr[5];- const int cols = (eg_ngram - 1) * eg_heads;- eg_map.resize((size_t) eg_vocab);- eg_mult.resize((size_t) eg_layers * eg_ngram);- eg_primes.resize((size_t) eg_layers * cols);- eg_off.resize((size_t) eg_layers * cols);- ok = std::fread(eg_map.data(), 4, eg_map.size(), e) == eg_map.size() &&- std::fread(eg_mult.data(), 8, eg_mult.size(), e) == eg_mult.size() &&- std::fread(eg_primes.data(), 8, eg_primes.size(), e) == eg_primes.size() &&- std::fread(eg_off.data(), 8, eg_off.size(), e) == eg_off.size();- }- std::fclose(e);- if (!ok) {- err = d + "/engram.bin: unreadable";- return false;- }- return true;- }-- std::string str(const std::string& k, const std::string& def = "") const {- auto it = kv.find(k);- return it == kv.end() ? def : it->second;- }- double num(const std::string& k, double def) const {- auto it = kv.find(k);- return it == kv.end() ? def : std::atof(it->second.c_str());- }- std::vector<int> list(const std::string& k) const {- std::vector<int> v;- std::stringstream ss(str(k));- std::string x;- while (std::getline(ss, x, ','))- if (!x.empty()) v.push_back(std::atoi(x.c_str()));- return v;- }- const Tensor* get(const std::string& n) const {- auto it = t.find(n);- return it == t.end() ? nullptr : &it->second;- }- bool read_at(int file, uint64_t off, uint64_t n, void* dst) const { // (any thread; side by side where pfds is)- if ((size_t) file < pfds.size() && pfds[(size_t) file] != os::kBadFd)- return os::pread_par(pfds[(size_t) file], dst, n, off);- return os::pread_full(fds[(size_t) file], dst, n, off);- }- bool read(const Tensor& x, void* dst) const { return read_at(x.file, x.off, x.bytes, dst); }- const uint8_t* at(int file, uint64_t off) const { return maps[(size_t) file].first + off; } // (gguf: mapped)-};--// ------------------------------------------------------------------ geometry-struct Geo {- int vocab = 0, dim = 0, nl = 0, heads = 0, hd = 0, rd = 0, q_lora = 0, o_lora = 0, o_groups = 0, n_exp = 0,- k_exp = 0, ff = 0, win = 0, hc = 0, sinkhorn = 0, ix_heads = 0, ix_hd = 0, ix_topk = 0, cand_src = -1,- cand_topk = 0, cand_blk = 0, orig_ctx = 0, eos = 1;- float eps = 1e-20f, hc_eps = 1e-6f, route_scale = 1.0f, lim = 0.0f;- double rope_theta = 1e4, comp_theta = 1e4, factor = 1.0, beta_fast = 32, beta_slow = 1;- std::vector<int> ratios, kv_src, ix_src, eg_layers;- // the DSpark drafter (the checkpoint's mtp.* blocks): a block of [seed, noise x (block - 1)] through n_mtp blocks- int n_mtp = 0, d_block = 0, d_noise = 0, d_rank = 0, d_n_exp = 0, d_k = 0;- std::vector<int> d_targets;-- void from(const Pack& p) {- vocab = (int) p.num("vocab_size", 0), dim = (int) p.num("hidden_size", 0);- nl = (int) p.num("num_hidden_layers", 0), heads = (int) p.num("num_attention_heads", 0);- hd = (int) p.num("head_dim", 0), rd = (int) p.num("qk_rope_head_dim", 0);- q_lora = (int) p.num("q_lora_rank", 0), o_lora = (int) p.num("o_lora_rank", 0);- o_groups = (int) p.num("o_groups", 0), n_exp = (int) p.num("n_routed_experts", 0);- k_exp = (int) p.num("num_experts_per_tok", 0), ff = (int) p.num("moe_intermediate_size", 0);- win = (int) p.num("sliding_window", 0), hc = (int) p.num("hc_mult", 0);- sinkhorn = (int) p.num("hc_sinkhorn_iters", 0), ix_heads = (int) p.num("index_n_heads", 0);- ix_hd = (int) p.num("index_head_dim", 0), ix_topk = (int) p.num("index_topk", 0);- cand_src = (int) p.num("candidate_source_layer_id", -1), cand_topk = (int) p.num("candidate_topk_blocks", 0);- cand_blk = (int) p.num("candidate_block_size", 0);- orig_ctx = (int) p.num("rope_scaling.original_max_position_embeddings", 0);- eos = (int) p.num("eos_token_id", 1);- eps = (float) p.num("rms_norm_eps", 1e-20), hc_eps = (float) p.num("hc_eps", 1e-6);- route_scale = (float) p.num("routed_scaling_factor", 1.0), lim = (float) p.num("swiglu_limit", 0.0);- rope_theta = p.num("rope_theta", 1e4), comp_theta = p.num("compress_rope_theta", 1e4);- factor = p.num("rope_scaling.factor", 1.0), beta_fast = p.num("rope_scaling.beta_fast", 32);- beta_slow = p.num("rope_scaling.beta_slow", 1);- ratios = p.list("compress_ratios"), kv_src = p.list("kv_source_layer_ids");- ix_src = p.list("index_source_layer_ids"), eg_layers = p.list("engram_layer_ids");- d_block = (int) p.num("dspark_block_size", 0), d_noise = (int) p.num("dspark_noise_token_id", 0);- d_rank = (int) p.num("dspark_markov_rank", 0), d_n_exp = (int) p.num("dspark_n_routed_experts", 0);- d_k = (int) p.num("dspark_num_experts_per_tok", 0), d_targets = p.list("dspark_target_layer_ids");- // the DSpark blocks: an EXL3 pack's, or a GGUF pack's from DeepSeek's checkpoint (tools/ds4/drafter_pack.py)- n_mtp = d_block > 0 && (int) ratios.size() > nl &&- (p.get("mtp.0.attn.wq_a.trellis") != nullptr || p.get("mtp.0.attn.wq_a.weight") != nullptr)- ? (int) ratios.size() - nl : 0;- }-};--// DeepSeek's precompute_freqs_cis frequencies (YaRN when original_seq_len > 0), in F32 as torch computes them-std::vector<float> rope_freqs(int dim, int original, double base, double factor, double beta_fast, double beta_slow) {- std::vector<float> f((size_t) dim / 2);- for (int i = 0; i < dim / 2; ++i) {- const float e = (float) (2 * i) / (float) dim;- f[(size_t) i] = 1.0f / std::pow((float) base, e);- }- if (original > 0) {- auto corr = [&](double rot) {- return dim * std::log(original / (rot * 2 * 3.14159265358979323846)) / (2 * std::log(base)); // (M_PI: not MSVC's)- };- const double low = std::max(std::floor(corr(beta_fast)), 0.0);- const double high = std::min(std::ceil(corr(beta_slow)), (double) dim - 1);- const float den = (float) std::max(high - low, 1e-3);- for (int i = 0; i < dim / 2; ++i) {- float ramp = ((float) i - (float) low) / den;- ramp = std::min(1.0f, std::max(0.0f, ramp));- const float smooth = 1.0f - ramp;- f[(size_t) i] = f[(size_t) i] / (float) factor * (1.0f - smooth) + f[(size_t) i] * smooth;- }- }- return f;-}--// ------------------------------------------------------------------ devices, layers-struct Dev {- int id = 0;- cudaStream_t st = nullptr;- cudaEvent_t ev = nullptr, ev2 = nullptr; // (ev2: the lookahead's guesses are on the host)- int* ridx_a = nullptr; // (Metal) the next layers' guessed picks (2 x kPicks)- float* rw_a = nullptr;- size_t free0 = 0, total = 0, weights = 0, experts = 0, caches = 0, scratch = 0;- std::vector<void*> allocs;- // a window's scratch, rows of up to kWin tokens (the routed experts' rows: kWin * kPicks)- float *h = nullptr, *h2 = nullptr, *x = nullptr, *qr = nullptr, *q = nullptr, *kvr = nullptr, *ckr = nullptr,- *cgr = nullptr, *latent = nullptr, *ikr = nullptr, *iq = nullptr, *iw = nullptr, *scores = nullptr,- *bm = nullptr, *o = nullptr, *oa = nullptr, *ao = nullptr, *mix = nullptr, *a_pre = nullptr,- *a_post = nullptr, *a_comb = nullptr, *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr,- *glog = nullptr, *rw = nullptr, *moe_out = nullptr, *sg = nullptr, *su = nullptr, *sh = nullptr,- *sh_out = nullptr, *cpu_part = nullptr, *eg_x = nullptr, *eg_kv = nullptr, *logits = nullptr,- *freqs_win = nullptr, *freqs_comp = nullptr, *xe = nullptr, *xg = nullptr, *xu = nullptr, *xhh = nullptr,- *ed = nullptr;- float* glog_a = nullptr; // lookahead: the next two layers' gate logits on this layer's FFN input (rows 1024 apart)- int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *tok = nullptr;- unsigned long long* plan_ptr = nullptr; // the decode path's VRAM experts (0: on the CPU lane)- uint8_t* eg_rows = nullptr;- uint16_t *emb_rows = nullptr, *ext = nullptr; // the window's embedding rows; the window attention's rows- AttnScratch attn_scr; // a few queries' selected compressed rows decoded (sparse_attn)- // the hand-off to the next device through pinned host memory (these cards copy peer to peer only through the- // driver's staging, ~1.7 ms a copy): h | pre | topk | cand- uint8_t* bounce = nullptr;- cudaEvent_t ev_out = nullptr;- float* tap = nullptr; // a window's stream means before a DSpark target layer- // threads / tensor parallelism: its group and its index in it, the group's all-reduces it took part in, xfer_put's- // block counter, the hand-off to the next group (host-mapped: h | pre | topk | cand, then its flag), the window- // MoE's grouping scratch- int grp = 0, mi = 0;- unsigned ar_seq = 0;- unsigned* xcnt = nullptr;- // the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;- // logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)- x3::Mat head;- float *norm_w = nullptr, *d_norm = nullptr, *logits_part = nullptr;- unsigned head_seq = 0;- // the drafter's markov bias over the head group (markov_share): this member's rows [mk_v0, mk_v0 + mk_vn) of the- // markov head, the previous token's E row, the drafted tokens (the lead's: d_ints)- uint16_t *mk_share = nullptr, *mk_erow = nullptr;- int* mk_ints = nullptr;- int mk_v0 = 0, mk_vn = 0;- uint8_t* hand = nullptr;- unsigned* hand_flag = nullptr;- unsigned long long* grp_ptr = nullptr;- int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;-};--// the chunked prompt path's buffers on a device: a chunk of C tokens through a layer at once (the attention in slabs of-// A queries, the indexer in slabs of AI), its GEMMs through FP16 reconstructions of the weights and cuBLAS-struct Chunk {- int C = 0, A = 0, AI = 0;- float *h = nullptr, *x = nullptr, *acc = nullptr, *qr = nullptr, *kvr = nullptr, *ckr = nullptr, *cgr = nullptr,- *latent = nullptr, *ikr = nullptr, *q = nullptr, *o = nullptr, *oa = nullptr, *iq = nullptr, *iw = nullptr,- *scores = nullptr, *bm = nullptr, *mix = nullptr, *a_pre = nullptr, *a_post = nullptr, *a_comb = nullptr,- *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr, *glog = nullptr, *rw = nullptr,- *sg = nullptr, *su = nullptr, *y = nullptr, *eg_x = nullptr, *eg_kv = nullptr;- uint16_t *x16 = nullptr, *qr16 = nullptr, *o16 = nullptr, *oa16 = nullptr, *lat16 = nullptr, *sh16 = nullptr,- *h16 = nullptr, *eg_x16 = nullptr, *ext = nullptr, *w16 = nullptr, *emb = nullptr;- size_t w16_elems = 0;- int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *bounds = nullptr, *row_tok = nullptr, *pos = nullptr;- uint8_t* eg_rows = nullptr;- int* bounds_pin = nullptr; // pinned: the sort's bounds, read back- // the CPU lane's experts streamed in: ring_n VRAM slots of ring_stride bytes in kParts parts, a batch a part,- // kParts batches in flight on the copy stream (filled / freed: each part's last copy, its last use)- static constexpr int kParts = 4;- uint8_t* ring = nullptr;- size_t ring_stride = 0;- int ring_n = 0;- cudaStream_t cs = nullptr;- cudaEvent_t filled[kParts] = {}, freed[kParts] = {};- // in place (file_experts): each streamed expert's file region read into pinned staging (ring_n slots of- // stage_stride bytes, a part's slots free again once its `filled` copies are done) by the readers, in parallel,- // and DMA'd from there - a copy from the files' pages goes through the driver's staging, one thread at ~1.3 GB/s- uint8_t* stage = nullptr;- size_t stage_stride = 0;- std::unique_ptr<Pool> readers;- cublasHandle_t blas = nullptr;- // host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each- // with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)- uint8_t* hand[2] = {nullptr, nullptr};- unsigned* hand_flag[2] = {nullptr, nullptr};- unsigned* hand_ack = nullptr;- unsigned arb_seq = 0; // the group's chunk all-reduces- // pinned, two chunks in flight: the embedding rows (the first layer's device), the Engram rows (an Engram layer's)- uint16_t* emb_host[2] = {nullptr, nullptr};- uint8_t* eg_host[2] = {nullptr, nullptr};- cudaEvent_t emb_ev[2] = {nullptr, nullptr}, eg_ev[2] = {nullptr, nullptr};- double stream_bytes = 0; // the experts streamed in, in all- // the flow (a GGUF's in-place experts, every layer of the device on it alone; DS4_FLOW=0: the batches above): a- // (chunk, layer)'s streamed experts are those it routes to - its likely ones (the profile's chance of a pick by the- // chunk's tokens) queued once the layer before it routed, read while that one computes, the rest of its picks once- // it routed itself; never an expert no token of the chunk picked unless it was a likely one. Each item read by io- // (unbuffered: the disk straight into its staging slot, every part of every item in flight at once - none waits- // for a batch) or taken from its RAM-tier slot as it lies, then DMA'd into the ring in order, item i into slot- // i % ring_n; the compute takes the items as they land, in runs of what has.- struct Item {- int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)- int mi = 0; // the device's member index in the layer's group- bool direct = false; // (Metal) read straight into its ring slot: no staging, no DMA- const uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once read- const uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)- uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there- int64_t after = -1; // (layer-major) the items up to this one read first- std::atomic<int> left{0}; // parts still being read- std::atomic<bool> bad{false};- };- bool flow = false;- bool direct = false; // (Metal, unified memory) the flow reads the files straight into the ring- int dev = -1; // the device (index into devs) this chunk state belongs to- std::unique_ptr<IoQueue> io;- std::deque<Item> items; // items[0] is item `base`; the indices below are absolute- size_t base = 0, n_read = 0, n_dma = 0, n_comp = 0;- uint64_t n_file = 0; // file items so far (staging slot = ordinal % S)- int S = 0; // staging slots, stage_stride apart- std::vector<int64_t> stage_item; // staging slot -> the item last read into it (-1: none)- std::vector<cudaEvent_t> stage_ev, ring_fill, ring_free; // a staging slot's DMA out; a ring slot's DMA in, its use- int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where- size_t spec_begin = 0;- std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes- int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count- int *bc_pin = nullptr, *bc_dev = nullptr;- uint64_t n_batch = 0;- double wait_ms = 0; // the compute waiting for a read- uint64_t it_file = 0, it_ram = 0, it_spec = 0, it_spec_unused = 0;- uint64_t io_d0 = 0, io_b0 = 0; // io's unbuffered / buffered bytes when the run started- // layer-major (a run of several chunks, every layer on this device): the layers one after another, each through- // all the chunks; a chunk's state between its layers (h | pre | topk | cand, chunk k at k chunk_hand_bytes()) in- // host memory, and the layer cache - RAM slots where the experts a layer's chunk reads from the files are kept for- // its later chunks, so each is read once a run- bool lm = false, lm_pinned = false;- uint8_t* lm_state = nullptr;- size_t lm_state_bytes = 0;- uint8_t* lc = nullptr; // lc_bytes of slots, a layer's experts' blobs side by side- size_t lc_bytes = 0;- int lc_il = -1, lc_used = 0, lc_n = 0; // the layer the cache holds, its slots taken, its slots in all- size_t lc_first = 0, lc_chk = 0; // the layer's first item; the items before lc_chk all read- std::vector<int> lc_of; // expert -> its slot (-1: none)- std::vector<int64_t> lc_writer; // slot -> the item reading its expert into it- uint64_t it_lc = 0; // items taken from the layer cache-};--struct Layer {- int il = 0, dev = 0, ratio = 0;- // its group (tensor parallel: nm members, each a copy of the device state in mates, member mi's share of the- // heads, of wo_a's groups, of wo_b's rows and of the FFNs: gate / up columns, down rows; its part of an expert- // laid out as hlay, hblob bytes - the host state, the CPU lane and the full blob layout stay with member 0)- int grp = 0, nm = 1, mi = 0;- std::vector<Layer> mates;- x3::ExpertLayout hlay;- size_t hblob = 0;- int ff0 = 0, ffn = 0; // this member's FFN columns [ff0, ff0 + ffn) (split128 of n_ff; the whole one alone)- // this member's heads [ah0, ah0 + ahn) and wo_a groups [og0, og0 + ogn) - all of them alone; in a group of more- // members than the heads split among evenly (head_members: 9, 7 or 6 over 64 heads), the first ones none: their- // attention adds nothing to the sum- int ah0 = 0, ahn = 0, og0 = 0, ogn = 0;- // the chunked prompt path: each expert's virtual slot (its VRAM slot, or n_slots + its place among the CPU lane's- // experts, which a chunk streams in that order) - pinned on member 0, a device copy on each member- int* vs_pin = nullptr;- int* vs_dev = nullptr;- std::vector<int> host_list;- std::string name; // "layers.N." or, for the DSpark drafter's blocks, "mtp.N."- int n_exp = 0, k_exp = 0; // routed experts, and how many a token takes (the drafter's: 128, 3)- bool mtp = false; // a DSpark block (index stage = il - n_layers)- bool kv_src = false, ix_src = false, owns_k = false, cand_src = false, uses_cand = false;- int src = -1; // the kv source whose compressed KV this layer reads (index into Impl::srcs)- int ik_src = -1; // the kv source whose index keys its indexer reads- int eg = -1; // engram layer index, or -1- x3::Mat wq_a, wq_b, wkv, wo_a[16], wo_b, s1, s2, s3, c_wkv, c_wgate, i_wq_b, i_wk, eg_wkv;- float *q_norm = nullptr, *kv_norm = nullptr, *attn_norm = nullptr, *ffn_norm = nullptr, *c_norm = nullptr,- *ik_norm = nullptr, *sink = nullptr, *hca_fn = nullptr, *hca_scale = nullptr, *hca_base = nullptr,- *hcf_fn = nullptr, *hcf_scale = nullptr, *hcf_base = nullptr, *gate_bias = nullptr, *eg_w = nullptr,- *gate_bias_vl = nullptr; // (the image tokens' routing bias, with the vision tower)- uint16_t* i_wproj = nullptr;- float* gate_w = nullptr; // the router's weights, F32 (DeepSeek's inference computes the gate in F32)- uint8_t* win = nullptr; // [win] rows of kKvFp8 (kv_row_bytes)- float *c_kv = nullptr, *c_score = nullptr; // the compressor's partial group (ratio > 1)- // a run of chunks' mid-run checkpoint (feed_mark): the ring and the partial group as they were before a row- uint8_t* snap_win = nullptr;- float *snap_kv = nullptr, *snap_sc = nullptr;- // routed experts- x3::ExpertLayout lay;- size_t blob = 0; // bytes of one expert's blob- int n_vram = 0;- bool cpu_only = false; // a role's type has no device kernel: every expert of this layer is the CPU lane's- uint8_t* vram_base = nullptr;- std::vector<int> slot; // expert -> VRAM slot or -1- std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)- std::vector<uint8_t*> host; // expert -> host (non-null: on the CPU lane; its halves in hh)- // the CPU lane keeps an expert as NH column parts, part X on NUMA node X: gate / up columns [X n_ff / NH, ..),- // down columns [X dim / NH, ..), each a standalone EXL3 matrix (its own trellis, the full suh, its part of svh)- struct Half {- size_t trellis[3], suh[3], svh[3], bytes;- int k[3], n[3];- } hl;- std::vector<std::array<uint8_t*, 4>> hh; // expert -> its parts on the nodes- // the expert cache: one VRAM slot and one set of host parts kept free, so an expert can move in while the one it- // replaces moves out; heat is how much each expert was used lately (the profile's share to start with)- int n_slots = 0, spare_vram = -1;- std::array<uint8_t*, 4> spare_host = {nullptr, nullptr, nullptr, nullptr};- std::vector<float> heat;- std::vector<int> cpu_hits; // CPU-lane experts used since the cache last looked- bool swap_pending = false;- unsigned long long* tab_pin = nullptr; // pinned: the table entries a swap writes- // a speculative window's backups (trunk layers): the ring rows it replaced, and a compressor's state before it and- // the window's raw rows (kv, then score) to replay the accepted ones- uint8_t* ring_bak = nullptr;- float *c_snap = nullptr, *c_rows = nullptr;- unsigned long long* vram_tab = nullptr; // device [n_exp]- std::vector<uint8_t> zc; // (the RAM store) expert -> 1: its store slot is GPU-visible, the device computes it over PCIe- unsigned long long* zc_tab = nullptr; // device [n_exp]: those slots' device addresses (0: none) - the decode token's zc_stage reads it- std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob- std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)- std::vector<uint64_t> region_bytes;- std::vector<int> region_file;- size_t piece_src[3][3]; // part (gate, up, down) x (trellis, suh, svh): offset in the region- size_t piece_bytes[3][3];- size_t dense_bytes = 0;- // a GGUF's MXFP4 experts (lay.fmt == gg::kFmtMXFP4): matrix r (gate, up, down) of expert e lies at mx_off[r] +- // e mx_stride[r] in file mx_file[r] - loaded from there, and read in place (the file's mapping) by the CPU lane- int mx_file[3] = {-1, -1, -1};- uint64_t mx_off[3] = {0, 0, 0}, mx_stride[3] = {0, 0, 0};-};--struct Src {- int layer = 0, ratio = 1;- int64_t cap = 0;- // per device (nullptr where no layer reads it): rows of kKvFp4 (the compressed KV), kKvFp4p (the index keys)- std::vector<uint8_t*> ckv, ik;- uint8_t *h_ckv = nullptr, *h_ik = nullptr; // pinned: a window's new rows, on their way to the other devices- cudaEvent_t ev_ckv = nullptr, ev_ik = nullptr;- // threads: whether devices outside the source's group read it, and the host-mapped hand-off of a window's new rows- // (ckv rows, then ik rows at kWin * hd) and its flag- bool remote = false;- uint8_t* x_stage = nullptr;- unsigned* x_flag = nullptr;- // pipelined chunks: a second staging slot (slot 1; slot 0 is x_stage), its flag, and the readers' acknowledgements- // [slot][device] (32 apart); readers: the devices outside the source's group that read it- uint8_t* x_stage1 = nullptr;- unsigned* x_flag1 = nullptr;- unsigned* x_ack = nullptr;- std::vector<int> readers;- uint8_t* stage(int slot) const { return slot ? x_stage1 : x_stage; }- unsigned* flag(int slot) const { return slot ? x_flag1 : x_flag; }-};--// a host thread that enqueues one device's work (threads mode)-struct Worker {- std::thread th;- std::mutex mu;- std::condition_variable cv;- std::function<void()> job;- bool busy = false, quit = false;- void start() {- th = std::thread([this] {- for (;;) {- std::function<void()> j;- {- std::unique_lock<std::mutex> lk(mu);- cv.wait(lk, [&] { return quit || job != nullptr; });- if (quit) return;- j = std::move(job);- job = nullptr;- }- j();- {- std::lock_guard<std::mutex> lk(mu);- busy = false;- }- cv.notify_all();- }- });- }- void submit(std::function<void()> f) {- {- std::lock_guard<std::mutex> lk(mu);- job = std::move(f);- busy = true;- }- cv.notify_all();- }- void wait() {- std::unique_lock<std::mutex> lk(mu);- cv.wait(lk, [&] { return !busy; });- }- bool idle() {- std::lock_guard<std::mutex> lk(mu);- return !busy;- }- void stop() {- {- std::lock_guard<std::mutex> lk(mu);- quit = true;- }- cv.notify_all();- if (th.joinable()) th.join();- }-};--struct Group {- std::vector<int> mem; // device indices, member order (member 0 the lead)- ArGroup ar, arb; // a window's all-reduces, a chunk's- // the head group: the members' logits shares on their way to the lead, two slots (by seq), their flags and the- // lead's acknowledgements (host-mapped)- float* lg_stage = nullptr;- unsigned *lg_flag = nullptr, *lg_ack = nullptr;-};--constexpr int kMaxL = 64; // layers (the trunk's and the drafter's) a window plan covers--// what a window's layers need from the host, decided before they run (threads enqueue the layers concurrently): the-// mailbox seq of each layer (async: the device publishes its routing), the indexer's selection sizes and the candidate-// blocks as each layer sees them, the window's hand-off seq-struct WinPlan {- bool par = false, async = false;- unsigned win = 0;- int k = 0; // (a run of chunks) the chunk's index: its hand-off slot k % 2- int snap_t = -1; // (a run of chunks) the chunk's row whose state before it the layers keep (-1: none)- int eg_slot = -1; // (a run of chunks, layer-major) the Engram rows' slot (-1: k % 2)- unsigned vl = 0; // (a window) bit t: token t is inside a picture's span (routed with the image bias)- unsigned seq[kMaxL] = {};- int topk[kMaxL][kWin] = {}, cand[kMaxL][kWin] = {};-};--} // namespace--// ------------------------------------------------------------------ the model-struct Model::Impl {- Options o;- Pack pk;- Geo g;- std::vector<Dev> devs;- std::vector<Layer> L;- std::vector<Src> srcs;- std::vector<Group> groups;- std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per device- bool par = false; // threads mode (the windows: decode)- bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads asked- unsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)- // the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows- int chunk = 0, chunk_min = 512;- static constexpr int kFlowChunkMin = 2; // chunk_min with every device's layers on the flow (see load)- // --prefill-chunk at most (route() runs in kVlRows-row slabs). The default stays kVlRows: a 16384-token chunk- // reads a 16K prompt's experts once instead of twice (V100: 461.8 -> 321.3 s, 736 -> 506 GB read) but its buffers- // take ~1.9 GB of VRAM experts (745 -> 644 on a 32 GB V100) from every decoded token- static constexpr int kMaxChunk = 4 * kVlRows;- std::vector<std::unique_ptr<Chunk>> chunks; // per device- size_t eg_cap = kWin; // tokens h_eg holds- int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)- bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)- std::vector<float> chunk_logits;- std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)- std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer- std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded- // a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this- // PC's speeds measured at load (calibrate_paths), each path's predictions scaled by how long it really took- struct PathModel {- bool on = false;- double dma = 0, copy = 0, disk = 0, cpu = 0, cpu_cold = 0; // bytes/s (disk, cpu_cold 0: not measured)- // the flow's taken / predicted so far [0] by the prompt's size class (2-31, 32-127, 128-511, 512 tokens and- // more: a short one's fixed costs weigh more), and how many prompts set it ([1] unused)- double k[2][4] = {{1, 1, 1, 1}, {1, 1, 1, 1}};- int64_t kn[2][4] = {};- double win_s = 0; // windows: seconds a window took, so far- int64_t win_n = 0;- int last_cls = 0, last_nw = 0; // the last pick's size class, its windows- double m = 1; // the picks' spread, against the profile's- int64_t n_m = 0;- int last = -1; // the last pick: 1 the flow, 0 windows- double last_pred = 0; // its prediction (s, before its k)- } pm;- static constexpr int kPickMax = 2048; // (from here on the flow: windows' work grows with every 16 tokens)- static constexpr double kPickRes = 0.9; // (this share of the CPU-lane experts in RAM: windows if predicted faster)- std::atomic<int> lm_layers{0}; // (layer-major) the layers every chunk went through- WinPlan wplan;- int head_dev = 0;- x3::Mat head;- float* norm_w = nullptr; // on the head device- // the drafter: main_proj / main_norm (on its first block's device), its final norm, the markov head and the- // confidence head (on the head device: the drafts' logits come from the trunk's head)- x3::Mat d_main_proj;- float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;- uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]- // a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its- // share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole- // rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq- bool mk_split = false;- uint16_t* h_mk_embed = nullptr;- unsigned* mk_slots = nullptr;- unsigned mk_seq = 0;- int d_dev = 0; // the drafter's first block's device- bool spec_win = false; // the window being run is a draft's verification (layer() keeps backups)- float* h_taps = nullptr; // pinned [kWin][targets][dim]: the last window's taps- cudaEvent_t ev_seed = nullptr; // the drafter's seeding read h_taps: the next window's taps wait for it- float* d_confv = nullptr; // on the head device- int* d_ints = nullptr; // on the head device: drafts in, picks out- float conf_threshold = std::getenv("DS4_DSPARK_CONF") ? (float) std::atof(std::getenv("DS4_DSPARK_CONF")) : 0.5f;- // a sampled round's drafts: the distribution q each one was drawn from (its kept list, on the head device) for- // verify()'s rule - a point at the draft for the drafter's argmax; and the chain they were drawn with (verify()- // checks it is its own)- strata::dsv41k::SpecList* d_specq = nullptr;- std::vector<strata::dsv41k::SpecList> h_specq = std::vector<strata::dsv41k::SpecList>(kWin);- bool drafts_q = false;- float dq_temp = 0.0f, dq_top_p = 1.0f, dq_min_p = 0.0f;- int dq_top_k = 0;- uint64_t dq_seed = 0, dq_counter = 0;- // (tests) DS4_DSPARK_SAMPLE=1: a sampled request's drafts drawn from the drafter's distribution through the- // request's chain, as DeepSeek's reference does (else its argmax: the same acceptance on Uranus, 31-32% of 5- // drafts a round either way); DS4_DSPARK_QTEMP=f: those draws at f x the request's temperature (any q keeps the- // rule exact); DS4_DSPARK_MAX=n: at most n drafts a round; DS4_DSPARK_DUMP=file: the drafter's inputs and outputs- // (tools/ds4/dspark_ref.py checks them against DeepSeek's checkpoint)- bool sample_drafts = std::getenv("DS4_DSPARK_SAMPLE") != nullptr && std::atoi(std::getenv("DS4_DSPARK_SAMPLE")) != 0;- float q_temp_scale = std::getenv("DS4_DSPARK_QTEMP") ? (float) std::atof(std::getenv("DS4_DSPARK_QTEMP")) : 1.0f;- int max_drafts = std::getenv("DS4_DSPARK_MAX") ? std::atoi(std::getenv("DS4_DSPARK_MAX")) : 0;- std::FILE* dump_d = std::getenv("DS4_DSPARK_DUMP") ? std::fopen(std::getenv("DS4_DSPARK_DUMP"), "wb") : nullptr;- float* h_dump = nullptr; // (the dump's pinned copies: each block's stream, each block's ring)- uint16_t* h_dump_ring = nullptr;-- uint16_t* embed_pinned = nullptr;- std::vector<float> freqs_win, freqs_comp;- std::vector<std::unique_ptr<Pool>> pools; // one per NUMA node, its workers on that node's cores- std::vector<std::vector<int>> cores; // each node's physical cores- int NH = 1; // the CPU lane's parts of an expert (the NUMA nodes)- // the CPU lane's experts read in place from the model's files (one part, the files' layout; the page cache keeps- // what fits) instead of copied into RAM, and how much of them the load reads ahead (the most-routed first)- bool file_experts = false;- uint64_t warm_bytes = 0;- static constexpr uint64_t kRamMargin = 8ull << 30; // RAM left to the rest (the system, the page cache's churn)- uint8_t* file_part(const Layer& l, int e) const {- return pk.maps[(size_t) l.region_file[(size_t) e]].first + l.region_off[(size_t) e];- }- // host side of the CPU lane, a window's rows- float *h_x = nullptr, *h_w = nullptr, *h_part = nullptr;- int* h_idx = nullptr;- int* h_idx_a = nullptr; // (Metal) the lookahead's guessed picks, 2 x kPicks- uint8_t* h_eg = nullptr; // the window's Engram rows: [engram layer][token] of [cols][hd] then the scales- // the Engram rows' reader (its own threads and file handles), the rows already fetched, the window's batches (one- // an Engram layer: asked for when the window's tokens are known, collected right before that layer)- EgReader eg_io;- EgRowCache eg_cache;- std::mutex eg_cache_mu;- std::vector<std::unique_ptr<EgBatch>> eg_win;- // Engram layer li's embed rows, as the pack stores them: 0 MXFP8 ([scale, 32 codes] blocks, 33 bytes a 32), 1- // ggml Q6_K (210 bytes a 256), 2 ggml IQ4_NL (18 bytes a 32: an fp16 d and 16 nibble bytes), 3 an old pack's FP8- // codes with their E8M0 scales in a tensor of their own (see eg_issue)- std::vector<uint8_t> eg_fmt;- std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};- uint16_t* h_emb = nullptr; // the window's embedding rows- std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;- // the state- int64_t pos = 0;- std::vector<int32_t> toks, ctoks;- struct Check { // the state at a position: every window ring, the compressors' partial groups- int64_t pos = -1;- std::vector<std::vector<uint8_t>> win;- std::vector<std::vector<float>> comp;- };- Check ck[2]; // the end of the last prompt, and the position before it- std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits- Stats st;- int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query- // (plan_window's: the windows' plans are made in order)- int n_logits = 0; // the last forward's logit rows- // the decode path (windows of one): the device publishes each layer's routing into reqs, the service thread- // answers in resps with the CPU lane's part, and the device waits for the answer only before the expert sum- Req* reqs = nullptr;- Resp* resps = nullptr;- unsigned seq_issued = 0;- std::atomic<int> layer_of[kRing];- std::thread service;- std::atomic<bool> quit{false};-#if defined(STRATA_METAL)- // (the async decode has the GPU wait on flags the CPU lane sets while it runs: the Metal queue cannot be waited on in the- // middle of a command buffer, so a decode step takes the host-routed path, as the prompt windows do)- bool sync_decode = std::getenv("DS4_ASYNC") == nullptr;-#else- bool sync_decode = std::getenv("DS4_SYNC") != nullptr;-#endif- // a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run- // before the wait for the CPU lane - the same bits, no faster)- bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;- int async_max_t = std::getenv("DS4_ASYNC_T") ? std::atoi(std::getenv("DS4_ASYNC_T")) : 8; // windows up to this- // lookahead: while layer L runs, the device guesses the next layer's picks (its gate on L's FFN input, the same- // choice) and publishes them with L's routing; the CPU lane asks for the guessed experts it will read from the files- // at once (a thread of its own, os::prefetch), so the SSD reads them while L computes instead of on the next- // layer's critical path. Only the page cache sees it - the outputs are the same. DS4_AHEAD=0: off, 2: also the- // layer after (from L's input too); DS4_AHEAD_EXTRA: guesses past the next layer's top k (< 0: fewer - the guess's- // picks by rank on Sun: 97 91 81 70 57 41 %, past the top 6 29 20 %: reads wasted on an SSD-bound PC).- int ahead_mode = std::getenv("DS4_AHEAD") ? std::atoi(std::getenv("DS4_AHEAD")) : 1;- int ahead_extra = std::getenv("DS4_AHEAD_EXTRA") ? std::atoi(std::getenv("DS4_AHEAD_EXTRA")) : 0;- // (DS4_AHEAD_ORDER=0: the CPU lane computes a layer's experts in expert order, not the ones asked for early first)- bool ahead_order = std::getenv("DS4_AHEAD_ORDER") == nullptr || std::atoi(std::getenv("DS4_AHEAD_ORDER")) != 0;- std::vector<uint8_t> ahead_fl; // [layer][1024]: 1 the top-k guess, 2 any guess, a layer before; 4 two layers before;- // 8 asked for (prefetch)- std::vector<uint8_t> ahead_rank; // [layer][1024]: 1 + the expert's rank in the guess a layer before (0: not guessed)- std::vector<unsigned> ahead_for; // [layer]: the request (seq) its flags are for- std::thread pf_th;- std::mutex pf_mu;- std::condition_variable pf_cv;- std::vector<std::pair<const void*, size_t>> pf_q;- // the expert cache: swaps prepared on threads of their own (DS4_CACHE_THREADS, 1 by default - 4 measured no faster- // on one V100 - more for a PC to try: each its staging buffers and- // copy streams), switched between windows- struct Swap {- int layer = 0, in = -1, out = -1;- int ram = -1; // (copy_mode, `in` held in RAM) the spare slot `out` is copied into from VRAM- };- static constexpr int kMaxSwaps = 8;- // the swap rule: a CPU-lane expert in when its heat passes the coldest VRAM one's x margin + bonus; heat x decay- // every 256 tokens; at most `inflight` swaps under way (DS4_CACHE_MARGIN / _BONUS / _DECAY / _INFLIGHT)- static float env_f(const char* k, float d) { return std::getenv(k) ? (float) std::atof(std::getenv(k)) : d; }- const float c_margin = env_f("DS4_CACHE_MARGIN", 1.25f), c_bonus = env_f("DS4_CACHE_BONUS", 1.0f),- c_decay = env_f("DS4_CACHE_DECAY", 0.9f);- const int c_inflight = std::max(1, (int) env_f("DS4_CACHE_INFLIGHT", (float) kMaxSwaps));- // the RAM tier (a GGUF's in-place experts, as GLM's: the hottest in VRAM, the next hottest here, the coldest in the- // files): RAM slots of one expert blob each, filled at load with the hottest experts VRAM does not hold, so the- // CPU lane reads them from RAM; between windows (cache_tick) a hotter expert from the files takes the place of the- // layer's coldest held one - read into a free slot by ram_th, switched while nothing reads them. Opt-in- // (STRATA_DS4_RAM_TIER=1): on a V100 with 64 GB of RAM it was slower than the page cache alone (1.4 against 2.2-2.4- // tokens/s sampled) - the cache follows a conversation's experts at once, the tier's moves lag behind.- struct RamSwap {- int layer = 0, in = -1, out = -1, slot = -1;- };- std::vector<uint8_t*> ram_slots, ram_chunks;- std::vector<size_t> ram_chunk_bytes;- bool ram_pinned = false; // the slots are pinned memory (the prompt path DMAs them from where they lie)- size_t flow_part[3] = {0, 0, 0}; // the flow's staging room for each part: its largest over the trunk's layers- std::vector<int> ram_free;- std::deque<RamSwap> ram_queue;- std::vector<RamSwap> ram_ready;- std::condition_variable ram_cv;- std::thread ram_th;- int ram_inflight = 0, ram_held = 0;- bool ram_tier = std::getenv("STRATA_DS4_RAM_TIER") != nullptr && std::atoi(std::getenv("STRATA_DS4_RAM_TIER")) != 0;- // STRATA_DS4_DISK_SKIP_W (experimental opt-in, CHANGES THE ANSWERS; 0 = off): on the small windows (decode, a- // draft's check - never the prompt path) the CPU lane leaves out a routed expert whose share of its token's route- // weights is below this and whose pages are not in RAM now (the page cache or the RAM tier; os::in_ram - Linux every- // page, Windows sampled pages of the working set), so a token waits on the SSD only for the experts that weigh; the- // weight it carried is lost (no renormalisation). On layers from STRATA_DS4_DISK_SKIP_FROM (0) on. An expert left- // out still counts as used (the expert cache, the profile).- const float disk_skip_w = [] {- const char* v = std::getenv("STRATA_DS4_DISK_SKIP_W");- return v != nullptr ? std::max(0.0f, (float) std::atof(v)) : 0.0f;- }();- const int disk_skip_from = [] {- const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");- return v != nullptr ? std::max(0, std::atoi(v)) : 0;- }();- // the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)- std::unique_ptr<ExpertStore> store;- uint8_t* zc_stage_buf = nullptr; // device: k staging slots for the store's experts of a decode token- size_t zc_stride = 0;- bool zc_copy = true; // DS4_STORE_STAGE=0: the expert kernels read the arena over PCIe themselves- bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;- if (l.lay.fmt == gg::kFmtMXFP4) {- for (int r = 0; r < 3; ++r)- if (!os::in_ram(mx_mat(l, e, r), mx_part_bytes(l, r))) return false;- return true;- }- return !file_experts || os::in_ram(file_part(l, e), l.blob);- }- // a GGUF's CPU-lane experts copied into RAM (copy_experts, after VRAM took its share: --mmap-experts off - auto reads- // them in place; DS4_COPY_GB=<n>: the hottest n GB, the rest in place - static, no moves between- // them): RAM slots (l.ram, as the tier's) of NH parts, part X on NUMA node X with rows [X n / NH, (X + 1) n / NH) of- // each matrix at cp_lay[layer][r] - whole rows, so the split never cuts a GGUF block (MXFP4's 32, the IQ / K types'- // 256 values: blocks run along a row); one spare slot a layer for the cache's swaps (the expert VRAM gives back takes- // the RAM place of the one it takes in)- // Windows' commit on an integrated GPU: its allocations are charged to the commit (RAM + page file) like any private- // memory - measured on a Radeon 8065S with 160 GiB of its own: the system's free commit fell by each one, whatever- // its size (one above ~3 GiB also counts as the process's private bytes, a smaller one does not, the commit the- // same), so a page file at least the GPU's share is what lets it hold experts at all. The experts it holds- // (place_experts) and the copies stay within what Windows can still commit, less max(8 GB, 10% of the RAM) for the- // other programs - commit_kept: the experts that kept off the GPU, pagefile_hint: the page file (GB) that would hold- // them (the log's WARNING, a NOTE and INFO for the server)- double commit_kept = 0, pagefile_hint = 0;- double commit_margin() const {- const os::Commit c = os::commit();- return std::max(8.0 * (double) (1ull << 30), c.ram == ~uint64_t{0} ? 0.0 : 0.1 * (double) c.ram);- }- // what may still be committed beside the margin (~inf: nothing limits it; DS4_COMMIT_CAP_GB: at most that, tests)- double commit_room() const {- const os::Commit c = os::commit();- double avail = c.avail == ~uint64_t{0} ? 1e300 : (double) c.avail;- if (const char* v = std::getenv("DS4_COMMIT_CAP_GB")) avail = std::min(avail, std::atof(v) * 1e9);- return avail >= 1e300 ? 1e300 : std::max(0.0, avail - commit_margin());- }- // a device whose allocations Windows charges to the commit: an integrated GPU (DS4_COMMIT_CAP_GB: any, tests)- bool commit_charged(const Dev& d) const {- if (std::getenv("DS4_COMMIT_CAP_GB") != nullptr) return true;-#if defined(_WIN32)- int v = 0;- if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) != cudaSuccess) {- (void) cudaGetLastError();- return false;- }- return v != 0;-#else- (void) d;- return false;-#endif- }- bool copy_mode = false;- std::vector<std::array<uint8_t*, 2>> ram_part; // slot -> its parts- std::vector<std::array<size_t, 4>> cp_lay; // layer -> part offsets of gate, up, down rows; a part's bytes- std::vector<int> copy_spare; // layer -> its spare slot (-1: none)- std::mutex sw_mu;- std::condition_variable sw_cv;- std::deque<Swap> sw_queue;- std::vector<Swap> sw_ready;- struct SwapBufs {- uint8_t *a = nullptr, *b = nullptr, *h = nullptr;- std::vector<cudaStream_t> st; // a copy stream a device- };- std::vector<SwapBufs> sw_bufs;- std::vector<std::thread> cache_ths;- int sw_inflight = 0;- int64_t decay_tokens = 0;- bool check_swaps = std::getenv("DS4_CHECK_SWAPS") != nullptr;- std::vector<float> host_logits;- std::vector<void*> host_allocs;- std::FILE* dump_h = std::getenv("DS4_DUMP_H") ? std::fopen(std::getenv("DS4_DUMP_H"), "wb") : nullptr;- // DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or- // chunk path (tests: where the two part)- std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;- // DS4_DUMP_S: the sub-steps inside the layers named by DS4_DUMP_S_LAYERS (default 0,1,2,3), records- // [char name[16], int64 il, p0, T, nbytes][nbytes raw bytes] (window path, the same on CUDA and Metal)- std::FILE* dump_s = std::getenv("DS4_DUMP_S") ? std::fopen(std::getenv("DS4_DUMP_S"), "wb") : nullptr;- std::vector<int> dump_s_layers = [] {- std::vector<int> v;- const char* e = std::getenv("DS4_DUMP_S_LAYERS");- std::string str = e ? e : "0,1,2,3";- size_t a = 0;- while (a < str.size()) {- size_t b = str.find(',', a);- if (b == std::string::npos) b = str.size();- if (b > a) v.push_back(std::atoi(str.substr(a, b - a).c_str()));- a = b + 1;- }- return v;- }();- // DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),- // then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the- // slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias- // [n_exp] (F32)- std::FILE* dump_r = std::getenv("DS4_DUMP_R") ? std::fopen(std::getenv("DS4_DUMP_R"), "wb") : nullptr;- // DS4_DUMP_Y=<layer>:<file> (layer -1: every layer): its expert sets on the chunk path, a record a piece [layer,- // row0, rows, n_exp, wbase kind (0 VRAM, 1 ring), max_rows] (int64), its bounds [n_exp + 1], its h rows [rows ff]- // (F16) and y rows [rows dim] (F32)- int dump_y_layer = std::getenv("DS4_DUMP_Y") ? std::atoi(std::getenv("DS4_DUMP_Y")) : -1;- std::FILE* dump_y = [] {- const char* v = std::getenv("DS4_DUMP_Y");- const char* f = v != nullptr ? std::strchr(v, ':') : nullptr;- return f != nullptr ? std::fopen(f + 1, "wb") : nullptr;- }();- std::mutex dump_mu;- // DS4_PROF: the GPU's time a decoded token by stage - events in stream order at a layer's start, after its- // attention, after the routing and the shared expert, after the experts' gate / up, after the wait for the CPU- // lane, after the down pass and the post - read after the token's sync, each the time since the mark before it- bool prof_on = std::getenv("DS4_PROF") != nullptr;- std::FILE* trace_f = std::getenv("DS4_TRACE") != nullptr ? std::fopen(std::getenv("DS4_TRACE"), "wb") : nullptr;- struct ProfMark {- const Dev* d;- cudaEvent_t ev;- int stage;- };- std::mutex prof_mu;- std::vector<ProfMark> prof_marks;- std::map<int, std::vector<cudaEvent_t>> prof_pool; // by device: an event belongs to the device it was made on- bool prof_window = false; // (decode windows only: T == 1)- // ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared- // expert, after its experts (the streamed ones' waits for their reads in it); printed after the run- bool prof_chunk = false;- double cprof_ms[4] = {};- void prof_mark(const Dev& d, int stage) {- if (!prof_window && !prof_chunk) return;- std::lock_guard<std::mutex> lk(prof_mu);- int cur = 0;- cudaGetDevice(&cur);- cudaSetDevice(d.id); // (events of another device are an "invalid resource handle")- cudaEvent_t e = nullptr;- std::vector<cudaEvent_t>& pool = prof_pool[d.id];- if (pool.empty()) cudaEventCreate(&e);- else {- e = pool.back();- pool.pop_back();- }- cudaEventRecord(e, d.st);- cudaSetDevice(cur);- prof_marks.push_back({&d, e, stage});- }- void prof_collect() {- if (!prof_window && !prof_chunk) return;- std::lock_guard<std::mutex> lk(prof_mu);- std::vector<std::pair<const Dev*, cudaEvent_t>> last;- for (const ProfMark& m : prof_marks) {- auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });- if (it != last.end()) {- float ms = 0.0f;- cudaSetDevice(m.d->id);- if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)- (prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;- it->second = m.ev;- } else {- last.push_back({m.d, m.ev});- }- }- for (const ProfMark& m : prof_marks) prof_pool[m.d->id].push_back(m.ev);- prof_marks.clear();- if (!prof_chunk) ++st.prof_tokens;- }- void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {- std::vector<float> v((size_t) T * g.hc * g.dim);- cudaSetDevice(d.id);-#if defined(STRATA_METAL)- ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)- cudaDeviceSynchronize();-#endif- cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);- cudaStreamSynchronize(d.st);- std::lock_guard<std::mutex> lk(dump_mu);- const int64_t hdr[3] = {il, p0, T};- std::fwrite(hdr, 8, 3, dump_l);- std::fwrite(v.data(), 4, v.size(), dump_l);- std::fflush(dump_l);- }-- bool dump_s_on(int il) const {- if (dump_s == nullptr) return false;- for (int x : dump_s_layers)- if (x == il) return true;- return false;- }- bool dump_s_name(const char* name) const { // DS4_DUMP_S_NAMES=a,b,c: only these points (default all)- static const std::string names = [] { const char* e = std::getenv("DS4_DUMP_S_NAMES"); return std::string(e ? "," : "") + (e ? e : "") + (e ? "," : ""); }();- return names.empty() || names.find(std::string(",") + name + ",") != std::string::npos;- }- void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {- if (!dump_s_on(il) || !dump_s_name(name)) return;- std::vector<uint8_t> v(nbytes);- cudaSetDevice(d.id);-#if defined(STRATA_METAL)- ds4_metal_flush();- cudaDeviceSynchronize(); // (the batched dispatches have finished before the read)-#endif- cudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);- cudaStreamSynchronize(d.st);- dump_sub_host(name, il, p0, T, v.data(), nbytes);- }- void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {- if (!dump_s_on(il) || !dump_s_name(name)) return;- std::lock_guard<std::mutex> lk(dump_mu);- char nm[16] = {};- std::strncpy(nm, name, 15);- const int64_t hdr[4] = {il, p0, T, (int64_t) nbytes};- std::fwrite(nm, 1, 16, dump_s);- std::fwrite(hdr, 8, 4, dump_s);- std::fwrite(host, 1, nbytes, dump_s);- std::fflush(dump_s);- }-- ~Impl() {- eg_io.stop(); // (its threads write into h_eg)- for (auto& w : workers) w->stop();- for (size_t i = 0; i < chunks.size(); ++i)- if (chunks[i] && chunks[i]->blas) {- cudaSetDevice(devs[i].id);- cublasDestroy(chunks[i]->blas);- }- quit.store(true);- if (service.joinable()) service.join();- {- std::lock_guard<std::mutex> lk(pf_mu);- }- pf_cv.notify_all();- if (pf_th.joinable()) pf_th.join();- {- std::lock_guard<std::mutex> lk(sw_mu);- }- sw_cv.notify_all();- ram_cv.notify_all();- for (std::thread& t : cache_ths)- if (t.joinable()) t.join();- if (ram_th.joinable()) ram_th.join();- for (size_t i = 0; i < ram_chunks.size(); ++i) os::anon_free(ram_chunks[i], ram_chunk_bytes[i]);- for (Dev& d : devs) {- cudaSetDevice(d.id);- for (void* p : d.allocs) cudaFree(p);- if (d.st) cudaStreamDestroy(d.st);- if (d.ev) cudaEventDestroy(d.ev);- }- for (void* p : host_allocs) cudaFreeHost(p);- for (Layer& l : L)- for (uint8_t* b : l.host)- if (b != nullptr) {- (void) b; // host blobs live in one mapping per layer, unmapped below- }- for (auto& m : host_maps) {- if (chunk > 0) cudaHostUnregister(m.first);- os::anon_free(m.first, m.second);- }- }- std::vector<std::pair<void*, size_t>> host_maps;-- template <class T>- bool dalloc(Dev& d, T*& p, size_t n, std::string& err, size_t* acct = nullptr) {- void* v = nullptr;- cudaSetDevice(d.id);- const size_t b = align256(n * sizeof(T));- if (cudaMalloc(&v, b) != cudaSuccess) {- err = "cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";- return false;- }- // zeroed before anything lands in it: the loads copy in on the default stream, which the non-blocking d.st- // does not order against - a zeroing still queued there could wipe the weights just copied- if (cudaMemsetAsync(v, 0, b, d.st) != cudaSuccess || cudaStreamSynchronize(d.st) != cudaSuccess) {- cudaFree(v);- err = "zeroing " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";- return false;- }- d.allocs.push_back(v);- if (acct) *acct += b;- else d.scratch += b;- p = (T*) v;- return true;- }- static Layer& member(Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }- static const Layer& member(const Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }- // bytes of a VRAM slot of l's experts (a member's part of each, in a group)- static size_t unit(const Layer& l) { return l.nm > 1 ? l.hblob : l.blob; }- // the members of a group of nm that take heads: the most the heads and the wo_a groups split among evenly (64- // heads and 8 groups: 9 -> 8, 7 or 6 -> 4, 5 -> 4, 3 -> 2); the first members the rest, only their FFN share- int head_members(int nm) const {- int na = nm;- while (na > 1 && (g.heads % na != 0 || g.o_groups % na != 0)) --na;- return na;- }- // host-mapped memory (the same address on the host and every device), zeroed- template <class T>- bool hmap(T*& p, size_t n, std::string& err) {- void* v = nullptr;- if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {- err = "host-mapped memory (" + std::to_string((n * sizeof(T)) >> 20) + " MiB)";- return false;- }- std::memset(v, 0, n * sizeof(T) + 64);- host_allocs.push_back(v);- p = (T*) v;- return true;- }- template <class T>- bool halloc(T*& p, size_t n, std::string& err) {- void* v = nullptr;- if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocPortable) != cudaSuccess) {- err = "cudaHostAlloc " + std::to_string((n * sizeof(T)) >> 20) + " MiB failed";- return false;- }- host_allocs.push_back(v);- p = (T*) v;- return true;- }-- // ---- weights- bool upload(Dev& d, const Tensor& t, void* dst, std::string& err) {- std::vector<uint8_t> tmp(t.bytes);- if (!pk.read(t, tmp.data())) {- err = "read failed";- return false;- }- cudaSetDevice(d.id);- CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));- return true;- }- // a dense matrix in a ggml block type (x3::Mat fmt kFmtGG, row = k / 256 blocks): whole, or part of its rows / inputs- bool mat_gg(Dev& d, const std::string& pre, const Tensor& t, int ggt, x3::Mat& m, std::string& err, size_t* acct, int part,- int np, bool cols) {- if (t.shape.size() != 2 || t.shape[1] % 256 != 0) {- err = pre + ": a " + t.dtype + " tensor that is not a matrix of whole blocks";- return false;- }- int n = (int) t.shape[0], k = (int) t.shape[1];- const size_t rb = gg::type_row_bytes(ggt, k), bb = rb / ((size_t) k / 256);- std::vector<uint8_t> w((size_t) n * rb);- if (t.bytes != w.size() || !pk.read(t, w.data())) {- err = pre + ": read failed";- return false;- }- if (np > 1) {- const int whole = cols ? n : k, wd = whole / np;- if (whole % np != 0 || wd % (cols ? 1 : 256) != 0) {- err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " + std::to_string(np) + " parts";- return false;- }- std::vector<uint8_t> p;- if (cols) {- p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));- n = wd;- } else {- const size_t pb = (size_t) (wd / 256) * bb, off = (size_t) (part * wd / 256) * bb;- p.resize((size_t) n * pb);- for (size_t r = 0; r < (size_t) n; ++r) std::memcpy(p.data() + r * pb, w.data() + r * rb + off, pb);- k = wd;- }- w.swap(p);- }- uint8_t* dp = nullptr;- if (!dalloc(d, dp, w.size(), err, acct)) return false;- cudaSetDevice(d.id);- CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));- m = x3::Mat{};- m.fmt = gg::kFmtGG;- m.K = ggt;- m.raw = dp;- m.k = k;- m.n = n;- m.ld = (int) gg::type_row_bytes(ggt, k);- return true;- }- // a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its- // columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a- // standalone [n][k] matrix- bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,- int np, bool cols) {- const bool q8 = t.dtype == "Q8_0";- // a ggml K-quant dense matrix (Q4_K / Q5_K / Q6_K: 256 values a block; the GEMV path of ds4_iq)- const int ggt = (t.dtype == "Q4_K" || t.dtype == "Q5_K" || t.dtype == "Q6_K") ? gg_type_of(t.dtype) : -1;- if (ggt > 0) return mat_gg(d, pre, t, ggt, m, err, acct, part, np, cols);- if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {- err = pre + ": a " + t.dtype + " tensor that is not a matrix";- return false;- }- int n = (int) t.shape[0], k = (int) t.shape[1];- // a row's bytes, and the bytes of v of its values (BF16: 2 each; Q8_0: 34 every 32)- auto bytes_of = [&](size_t v) { return q8 ? v / 32 * gg::kQ8Bytes : v * 2; };- std::vector<uint8_t> w((size_t) n * bytes_of((size_t) k));- if (t.bytes != w.size() || !pk.read(t, w.data())) {- err = pre + ": read failed";- return false;- }- if (np > 1) {- const int whole = cols ? n : k, wd = whole / np;- if (whole % np != 0 || wd % (q8 ? 32 : 8) != 0) {- err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " +- std::to_string(np) + " parts";- return false;- }- const size_t rb = bytes_of((size_t) k);- std::vector<uint8_t> p;- if (cols) {- p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));- n = wd;- } else {- const size_t pb = bytes_of((size_t) wd);- p.resize((size_t) n * pb);- for (size_t r = 0; r < (size_t) n; ++r)- std::memcpy(p.data() + r * pb, w.data() + r * rb + bytes_of((size_t) part * wd), pb);- k = wd;- }- w.swap(p);- }- uint8_t* dp = nullptr;- if (!dalloc(d, dp, w.size(), err, acct)) return false;- cudaSetDevice(d.id);- CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));- m = x3::Mat{};- m.fmt = q8 ? gg::kFmtQ8_0 : gg::kFmtBF16;- m.raw = dp;- m.k = k;- m.n = n;- m.ld = q8 ? (int) bytes_of((size_t) k) : k;- return true;- }- // a Q8_0 tensor's values as F32 (host)- static std::vector<float> q8_values(const uint8_t* q, size_t n) {- std::vector<float> v(n);- for (size_t b = 0; b < n / 32; ++b) {- const uint8_t* p = q + b * gg::kQ8Bytes;- const float d = f16_to_f32((uint16_t) (p[0] | p[1] << 8));- for (int j = 0; j < 32; ++j) v[b * 32 + j] = d * (float) (int8_t) p[2 + j];- }- return v;- }- // an EXL3 matrix onto the device: whole, or (np > 1) part `part` of its columns (cols) or of its rows - a standalone- // matrix of its own (the rotations' 128-blocks stay whole)- // (rot: split128's - an EXL3 matrix's uneven 128-block shares going round the members; a GGUF's BF16 / Q8_0 matrix- // splits evenly, its groups only as wide as that goes)- bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,- bool cols = true, int rot = 0) {- if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0" || tw->dtype == "Q4_K" || tw->dtype == "Q5_K" || tw->dtype == "Q6_K"))- return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);- const Tensor* tr = pk.get(pre + ".trellis");- const Tensor* su = pk.get(pre + ".suh");- const Tensor* sv = pk.get(pre + ".svh");- if (tr == nullptr || su == nullptr || sv == nullptr || tr->shape.size() != 3) {- err = pre + ": not an EXL3 matrix in the pack";- return false;- }- if (pk.get(pre + ".mul1") == nullptr) {- err = pre + ": only the mul1 codebook is covered";- return false;- }- m.k = (int) tr->shape[0] * 16;- m.n = (int) tr->shape[1] * 16;- m.K = (int) tr->shape[2] / 16;- m.cb = 2;- m.ld = 0;- if (np > 1) {- const int whole = cols ? m.n : m.k;- int at = 0, w = 0;- if (whole % 128 != 0 || whole / 128 < np) {- err = pre + ": " + std::to_string(whole) + (cols ? " columns" : " rows") + " do not split in " +- std::to_string(np) + " parts of 128-blocks";- return false;- }- split128(whole, np, part, at, w, rot);- std::vector<uint8_t> t0(tr->bytes), u0(su->bytes), v0(sv->bytes);- if (!pk.read(*tr, t0.data()) || !pk.read(*su, u0.data()) || !pk.read(*sv, v0.data())) {- err = pre + ": read failed";- return false;- }- const size_t tile = tr->bytes / ((size_t) (m.k / 16) * (m.n / 16));- std::vector<uint8_t> t1, u1, v1;- if (cols) { // the tiles [c0 / 16, (c0 + w) / 16) of each 16-row slice, svh [c0, c0 + w)- const size_t c0 = (size_t) at, nt = (size_t) m.n / 16, nth = (size_t) w / 16;- t1.resize((size_t) (m.k / 16) * nth * tile);- for (size_t sl = 0; sl < (size_t) m.k / 16; ++sl)- std::memcpy(t1.data() + sl * nth * tile, t0.data() + (sl * nt + c0 / 16) * tile, nth * tile);- u1 = u0;- v1.assign(v0.begin() + (long) (c0 * 2), v0.begin() + (long) ((c0 + w) * 2));- m.n = w;- } else { // the 16-row slices [r0 / 16, (r0 + w) / 16), suh [r0, r0 + w)- const size_t r0 = (size_t) at, row = (size_t) (m.n / 16) * tile;- t1.assign(t0.begin() + (long) (r0 / 16 * row), t0.begin() + (long) ((r0 + w) / 16 * row));- u1.assign(u0.begin() + (long) (r0 * 2), u0.begin() + (long) ((r0 + w) * 2));- v1 = v0;- m.k = w;- }- uint32_t* dt = nullptr;- uint16_t *ds = nullptr, *dv = nullptr;- if (!dalloc(d, dt, t1.size() / 4, err, acct) || !dalloc(d, ds, u1.size() / 2, err, acct) ||- !dalloc(d, dv, v1.size() / 2, err, acct))- return false;- cudaSetDevice(d.id);- CK(cudaMemcpy(dt, t1.data(), t1.size(), cudaMemcpyHostToDevice));- CK(cudaMemcpy(ds, u1.data(), u1.size(), cudaMemcpyHostToDevice));- CK(cudaMemcpy(dv, v1.data(), v1.size(), cudaMemcpyHostToDevice));- m.trellis = dt;- m.suh = ds;- m.svh = dv;- return x3::supported(m) || (err = pre + ": a part not covered by the EXL3 kernels", false);- }- uint32_t* dt = nullptr;- uint16_t *ds = nullptr, *dv = nullptr;- if (!dalloc(d, dt, tr->bytes / 4, err, acct) || !dalloc(d, ds, su->bytes / 2, err, acct) ||- !dalloc(d, dv, sv->bytes / 2, err, acct))- return false;- if (!upload(d, *tr, dt, err) || !upload(d, *su, ds, err) || !upload(d, *sv, dv, err)) return false;- m.trellis = dt;- m.suh = ds;- m.svh = dv;- if (!x3::supported(m)) {- err = pre + ": k " + std::to_string(m.k) + " n " + std::to_string(m.n) + " K " + std::to_string(m.K) +- " not covered by the EXL3 kernels";- return false;- }- return true;- }- // a float tensor (F32 / F16 / BF16) as F32 on the device- bool f32(Dev& d, const std::string& name, float*& p, std::string& err, size_t* acct, size_t want = 0) {- const Tensor* t = pk.get(name);- if (t == nullptr) {- err = name + ": not in the pack";- return false;- }- std::vector<uint8_t> raw(t->bytes);- if (!pk.read(*t, raw.data())) {- err = name + ": read failed";- return false;- }- size_t n = 0;- std::vector<float> v;- if (t->dtype == "F32") {- n = t->bytes / 4;- v.resize(n);- std::memcpy(v.data(), raw.data(), t->bytes);- } else if (t->dtype == "F16" || t->dtype == "BF16") {- n = t->bytes / 2;- v.resize(n);- const uint16_t* s = (const uint16_t*) raw.data();- for (size_t i = 0; i < n; ++i) v[i] = t->dtype == "F16" ? f16_to_f32(s[i]) : bf16_to_f32(s[i]);- } else if (t->dtype == "Q8_0") { // (a GGUF's router may come so)- n = t->bytes / gg::kQ8Bytes * 32;- v = q8_values(raw.data(), n);- } else {- err = name + ": dtype " + t->dtype + " is not a float";- return false;- }- if (want != 0 && n != want) {- err = name + ": " + std::to_string(n) + " values, expected " + std::to_string(want);- return false;- }- if (!dalloc(d, p, n, err, acct)) return false;- cudaSetDevice(d.id);- CK(cudaMemcpy(p, v.data(), n * 4, cudaMemcpyHostToDevice));- return true;- }- bool f16raw(Dev& d, const std::string& name, uint16_t*& p, std::string& err, size_t* acct) {- const Tensor* t = pk.get(name);- if (t != nullptr && (t->dtype == "F32" || t->dtype == "BF16" || t->dtype == "Q8_0")) { // (a GGUF's)- const size_t n = t->dtype == "Q8_0" ? t->bytes / gg::kQ8Bytes * 32 : t->bytes / (t->dtype == "F32" ? 4 : 2);- std::vector<uint8_t> raw(t->bytes);- if (!pk.read(*t, raw.data())) {- err = name + ": read failed";- return false;- }- const std::vector<float> q = t->dtype == "Q8_0" ? q8_values(raw.data(), n) : std::vector<float>();- std::vector<uint16_t> h(n);- for (size_t i = 0; i < n; ++i) {- float v;- if (t->dtype == "F32") std::memcpy(&v, raw.data() + i * 4, 4);- else if (t->dtype == "Q8_0") v = q[i];- else v = bf16_to_f32(((const uint16_t*) raw.data())[i]);- h[i] = strata::kernels::f16_from_f32(v);- }- if (!dalloc(d, p, n, err, acct)) return false;- cudaSetDevice(d.id);- CK(cudaMemcpy(p, h.data(), n * 2, cudaMemcpyHostToDevice));- return true;- }- if (t == nullptr || t->dtype != "F16") {- err = name + ": not an F16 tensor in the pack";- return false;- }- if (!dalloc(d, p, t->bytes / 2, err, acct)) return false;- return upload(d, *t, p, err);- }-- size_t mat_bytes(const std::string& pre) const {- if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);- size_t b = 0;- for (const char* s : {".trellis", ".suh", ".svh"})- if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);- return b;- }- size_t tensor_bytes(const std::string& name) const {- const Tensor* t = pk.get(name);- return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;- }-- // ---- expert regions and blob layout- // a GGUF's MXFP4 experts: the blob is gate | up | down ([out][in / 32] blocks each, as the file holds them), every- // expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)- // (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)- static int gg_type_of(const std::string& dtype) {- for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 145, 146, 224, 225, 226, 227})- if (dtype == gg::type_name(t)) return t;- return -1;- }- bool expert_layout_mx(Layer& l, std::string& err) {- const std::string P = l.name + "ffn.experts.";- const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down- size_t off = 0;- for (int r = 0; r < 3; ++r) {- const int n = r < 2 ? g.ff : g.dim, k = r < 2 ? g.dim : g.ff;- const Tensor* t0 = pk.get(P + "0." + parts[r] + ".weight");- const Tensor* t1 = l.n_exp > 1 ? pk.get(P + "1." + parts[r] + ".weight") : t0;- const int ty = t0 != nullptr ? gg_type_of(t0->dtype) : -1;- const size_t bytes = ty > 0 ? (size_t) n * gg::type_row_bytes(ty, k) : 0;- if (t0 == nullptr || t1 == nullptr || bytes == 0 || t0->bytes != bytes) {- err = P + "0." + parts[r] + ": not an MXFP4 or low-bit [" + std::to_string(n) + "][" + std::to_string(k) +- "] matrix the engine reads (" + (t0 != nullptr ? t0->dtype : std::string("missing")) + ")";- return false;- }- l.mx_file[r] = t0->file;- l.mx_off[r] = t0->off;- l.mx_stride[r] = l.n_exp > 1 ? t1->off - t0->off : bytes;- for (int e = 0; e < l.n_exp; ++e) {- const Tensor* t = pk.get(P + std::to_string(e) + "." + parts[r] + ".weight");- if (t == nullptr || t->dtype != t0->dtype || t->bytes != bytes || t->file != l.mx_file[r] ||- t->off != l.mx_off[r] + (uint64_t) e * l.mx_stride[r]) {- err = P + std::to_string(e) + "." + parts[r] + ": missing, or not where the layer's others are";- return false;- }- }- l.lay.trellis[r] = off;- l.lay.suh[r] = l.lay.svh[r] = off;- l.lay.K[r] = ty;- off += align256(bytes) + (store_gb() > 0 ? ExpertStore::kSlack : 0); // (the store's aligned reads spill into the gaps)- }- l.lay.fmt = gg::kFmtMXFP4;- if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||- gg::expert_type(l.lay, 0) == gg::kTypeMXFP4 ||- gg::expert_type(l.lay, 2) == gg::kTypeMXFP4)) {- err = P + ": gate / up / down of types " + gg::type_name(l.lay.K[0]) + " / " + gg::type_name(l.lay.K[1]) +- " / " + gg::type_name(l.lay.K[2]) + " (gate and up must match; MXFP4 does not mix with the others)";- return false;- }- l.blob = off;- // a type the device kernels do not take (IQ1_S / IQ1_M / IQ4_NL today): the layer stays on the CPU lane, where- // ggml-cpu's row dots read it - one warning, then the placement code keeps it out of VRAM- for (int r = 0; r < 3; ++r)- if (!gg::type_on_device(l.lay.K[r])) {- l.cpu_only = true;- std::fprintf(stderr,- "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "- "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",- gg::type_name(l.lay.K[r]));- break;- }- return true;- }- const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's- if (store) {- const int s = store->find(l.il, e);- if (s >= 0) return store->mat(s, r);- }- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)- return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]- : ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];- return pk.at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r]);- }- size_t mx_part_bytes(const Layer& l, int r) const { // (its type's)- return r < 2 ? (size_t) g.ff * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)- : (size_t) g.dim * gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff);- }- // (copy_mode) matrix r's rows, and the rows of one NUMA part of it- int mx_n(int r) const { return r < 2 ? g.ff : g.dim; }- size_t mx_row_b(const Layer& l, int r) const { return mx_part_bytes(l, r) / (size_t) mx_n(r); }- // part X's rows [X n / NH, ..) of expert e's matrix r: its copy's part on node X, else where the file has them- const uint8_t* mx_part_ptr(const Layer& l, int e, int r, int X) const {- if (copy_mode && l.ram[(size_t) e] >= 0)- return ram_part[(size_t) l.ram[(size_t) e]][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];- return mx_mat(l, e, r) + (size_t) X * (size_t) (mx_n(r) / NH) * mx_row_b(l, r);- }- // a copy's NH parts -> the blob, and back (each matrix's rows of part X at X n / NH rows into the blob's matrix)- void ram_to_blob(const Layer& l, int s, uint8_t* blob) const {- for (int r = 0; r < 3; ++r) {- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);- for (int X = 0; X < NH; ++X)- std::memcpy(blob + l.lay.trellis[r] + (size_t) X * nb, ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], nb);- }- }- void blob_to_ram(const Layer& l, const uint8_t* blob, int s) const {- for (int r = 0; r < 3; ++r) {- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);- for (int X = 0; X < NH; ++X)- std::memcpy(ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], blob + l.lay.trellis[r] + (size_t) X * nb, nb);- }- }- // expert e (from the file) -> its blob; and member m of nm's part of a blob (gate / up rows [m n_ff / nm, ..), down- // inputs of the same range: in each row the blocks [m n_ff / nm / 32, ..)) and back- bool read_mx(const Layer& l, int e, uint8_t* blob) const { // (from its RAM-tier slot or copy when it has one)- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) {- if (copy_mode) ram_to_blob(l, l.ram[(size_t) e], blob);- else std::memcpy(blob, ram_slots[(size_t) l.ram[(size_t) e]], l.blob);- return true;- }- return read_mx_file(l, e, blob);- }- bool read_mx_file(const Layer& l, int e, uint8_t* blob) const { // (the movers: never a slot that may change)- for (int r = 0; r < 3; ++r)- if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r], mx_part_bytes(l, r),- blob + l.lay.trellis[r]))- return false;- return true;- }- void mx_blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {- const int ffh = g.ff / l.nm;- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);- for (int r = 0; r < 2; ++r)- std::memcpy(dst + l.hlay.trellis[r], blob + l.lay.trellis[r] + (size_t) m * ffh * gr, (size_t) ffh * gr);- for (size_t row = 0; row < (size_t) g.dim; ++row)- std::memcpy(dst + l.hlay.trellis[2] + row * dh, blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, dh);- }- void mx_half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {- const int ffh = g.ff / l.nm;- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);- for (int r = 0; r < 2; ++r)- std::memcpy(blob + l.lay.trellis[r] + (size_t) m * ffh * gr, src + l.hlay.trellis[r], (size_t) ffh * gr);- for (size_t row = 0; row < (size_t) g.dim; ++row)- std::memcpy(blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, src + l.hlay.trellis[2] + row * dh, dh);- }-- bool expert_layout(Layer& l, std::string& err) {- if (const Tensor* t = pk.get(l.name + "ffn.experts.0.w1.weight"); t != nullptr && gg_type_of(t->dtype) > 0)- return expert_layout_mx(l, err);- const std::string P = l.name + "ffn.experts.";- const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down- const char* pieces[3] = {".trellis", ".suh", ".svh"};- l.region_off.resize((size_t) l.n_exp);- l.region_bytes.resize((size_t) l.n_exp);- l.region_file.resize((size_t) l.n_exp);- size_t off = 0;- for (int r = 0; r < 3; ++r) {- const Tensor* tr = pk.get(P + "0." + parts[r] + ".trellis");- if (tr == nullptr || pk.get(P + "0." + parts[r] + ".mul1") == nullptr) {- err = P + "0." + parts[r] + ": not an EXL3 (mul1) matrix";- return false;- }- l.lay.K[r] = (int) tr->shape[2] / 16;- l.lay.trellis[r] = off;- off += align256(tr->bytes);- l.lay.suh[r] = off;- off += align256(pk.get(P + "0." + parts[r] + ".suh")->bytes);- l.lay.svh[r] = off;- off += align256(pk.get(P + "0." + parts[r] + ".svh")->bytes);- }- l.lay.cb = 2;- l.blob = off;- for (int e = 0; e < l.n_exp; ++e) {- const std::string E = P + std::to_string(e) + ".";- uint64_t lo = UINT64_MAX, hi = 0;- int file = -1;- for (int r = 0; r < 3; ++r)- for (int q = 0; q < 3; ++q) {- const Tensor* t = pk.get(E + parts[r] + pieces[q]);- if (t == nullptr) {- err = E + parts[r] + pieces[q] + ": missing";- return false;- }- if (file < 0) file = t->file;- if (t->file != file) {- err = E + ": split across files";- return false;- }- lo = std::min(lo, t->off);- hi = std::max(hi, t->off + t->bytes);- if (e == 0) l.piece_bytes[r][q] = t->bytes;- else if (t->bytes != l.piece_bytes[r][q]) {- err = E + parts[r] + ": not the shape of expert 0's";- return false;- }- }- l.region_off[(size_t) e] = lo;- l.region_bytes[(size_t) e] = hi - lo;- l.region_file[(size_t) e] = file;- for (int r = 0; r < 3; ++r)- for (int q = 0; q < 3; ++q) {- const size_t src = pk.get(E + parts[r] + pieces[q])->off - lo;- if (e == 0) l.piece_src[r][q] = src;- else if (src != l.piece_src[r][q]) {- err = E + ": the pieces lie differently from expert 0's";- return false;- }- }- }- return true;- }- // the region of expert e (read from the file) -> its blob- void repack(const Layer& l, const uint8_t* region, uint8_t* blob) const {- for (int r = 0; r < 3; ++r) {- const size_t dst[3] = {l.lay.trellis[r], l.lay.suh[r], l.lay.svh[r]};- for (int q = 0; q < 3; ++q) std::memcpy(blob + dst[q], region + l.piece_src[r][q], l.piece_bytes[r][q]);- }- }-- // the region of expert e -> its CPU-lane part X (columns [X n / NH, (X + 1) n / NH) of each matrix)- void repack_part(const Layer& l, const uint8_t* region, int X, uint8_t* dst) const {- for (int r = 0; r < 3; ++r) {- const size_t tile = (size_t) 32 * l.lay.K[r];- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;- const uint8_t* tr = region + l.piece_src[r][0];- for (int sl = 0; sl < k / 16; ++sl)- std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,- (size_t) nth * tile);- std::memcpy(dst + l.hl.suh[r], region + l.piece_src[r][1], (size_t) k * 2);- std::memcpy(dst + l.hl.svh[r], region + l.piece_src[r][2] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);- }- }-- // ---- loading- bool load(const Options& opt, std::string& err) {- o = opt;- const double t0 = now_ms();- if (!pk.load(o.pack, err)) return false;- g.from(pk);- if (g.dim == 0 || g.nl == 0 || (int) g.ratios.size() < g.nl) {- err = o.pack + "/ds4.txt: incomplete geometry";- return false;- }- if (g.hd != 512 || g.ix_hd != 128 || g.k_exp > 8 || g.hc > 8) {- err = "geometry not covered (head_dim " + std::to_string(g.hd) + ", index_head_dim " +- std::to_string(g.ix_hd) + ", experts per token " + std::to_string(g.k_exp) + ")";- return false;- }- freqs_win = rope_freqs(g.rd, 0, g.rope_theta, g.factor, g.beta_fast, g.beta_slow);- freqs_comp = rope_freqs(g.rd, g.orig_ctx, g.comp_theta, g.factor, g.beta_fast, g.beta_slow);-- // devices- int ndev = 0;- if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {- err = "no CUDA device";- return false;- }- std::vector<int> ids = o.gpus;- if (ids.empty())- for (int i = 0; i < ndev; ++i) ids.push_back(i);- devs.resize(ids.size());- for (size_t i = 0; i < ids.size(); ++i) {- Dev& d = devs[i];- d.id = ids[i];- CK(cudaSetDevice(d.id));- CK(cudaStreamCreateWithFlags(&d.st, cudaStreamNonBlocking));- CK(cudaEventCreateWithFlags(&d.ev, cudaEventDisableTiming));- CK(cudaEventCreateWithFlags(&d.ev2, cudaEventDisableTiming));- if (!x3::prepare(d.st) || (pk.gguf && !gg::prepare(d.st))) {- err = "exl3 scratch on CUDA" + std::to_string(d.id);- return false;- }- CK(cudaMemGetInfo(&d.free0, &d.total));- }- // the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each- // expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)- int tp = o.tp;- if (pk.gguf && tp > 1) {- int blk = 32;- for (const auto& kv : pk.t) {- const std::string& n = kv.first;- if (n.find(".ffn.experts.") == std::string::npos || n.size() < 9 ||- n.compare(n.size() - 9, 9, "w2.weight") != 0)- continue;- const int ty = gg_type_of(kv.second.dtype);- if (ty >= 0) blk = std::max(blk, gg::type_block(ty));- }- while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % (std::getenv("DS4_TP_ALIGN") ? std::atoi(std::getenv("DS4_TP_ALIGN")) : 128) != 0 || g.heads % tp != 0 ||- g.o_groups % tp != 0))- --tp;- if (tp != o.tp)- std::fprintf(stderr, "ds4: tensor parallel %d -> %d: the experts' %d-value blocks split n_ff %d no other way "- "(each GPU its own layers)\n", o.tp, tp, blk, g.ff);- }- // the groups: runs of tp devices of one kind (their memory), a device unlike the next alone- // - or, tp the number of GPUs, one group of all of them whatever their kind (its tightest member sets the- // experts' room); threads mode for the windows (decode) where a group is 2 wide or more or --threads on, the- // prompt path (its own thread a GPU) as asked for - also where the experts' blocks took the groups' width down- // to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16- par = o.threads || tp > 1;- prompt_par = par || o.tp > 1;-#if defined(STRATA_METAL)- if (std::getenv("DS4_NO_CHUNKS") == nullptr) prompt_par = true; // the chunked prompt path on one device, the decode steps stay on the host-routed path-#endif- const bool tp_all = tp > 1 && tp == (int) devs.size();- for (size_t i = 0; i < devs.size();) {- Group G;- G.mem.push_back((int) i);- size_t j = i + 1;- while ((int) G.mem.size() < tp && j < devs.size() && (tp_all || devs[j].total == devs[i].total))- G.mem.push_back((int) j++);- if ((int) G.mem.size() < tp && G.mem.size() > 1) { // a short run at the end: devices of their own- for (int m : G.mem) {- Group one;- one.mem.push_back(m);- groups.push_back(one);- }- } else groups.push_back(G);- i = j;- }- for (size_t gi = 0; gi < groups.size(); ++gi)- for (size_t m = 0; m < groups[gi].mem.size(); ++m) {- devs[(size_t) groups[gi].mem[m]].grp = (int) gi;- devs[(size_t) groups[gi].mem[m]].mi = (int) m;- }- // whether the CPU lane's experts are read in place (--mmap-experts; auto: the pack's expert bytes beyond what the- // GPUs can take - their free memory less the other weights and the prompt path's buffers - do not fit the RAM- // this process can get beside kRamMargin), decided here because it sets the prompt chunk's default: in place- // every chunk streams every expert VRAM does not hold from the files, so the larger the chunk the fewer times- if (pk.gguf) file_experts = true; // (a GGUF's: the files stay their source - copy_experts copies them into RAM- // after VRAM's are in, by --mmap-experts)- else if (o.mmap_experts >= 0) file_experts = o.mmap_experts > 0;- else {- double exp_b = 0, other_b = 0, room = 0;- for (const auto& kt : pk.t) {- const std::string& n = kt.first;- if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||- (!o.dspark && n.rfind("mtp.", 0) == 0))- continue;- (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;- }- for (const Dev& d : devs)- room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) -- (prompt_par ? 1.5 : 0.4) * (1 << 30)) *- o.vram_experts;- file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();- }- chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : file_experts ? kVlRows : 2048) : 0;- const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)- if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;- if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));- if (chunk > 0) {- chunk = std::min(kMaxChunk, std::max(dc != nullptr ? 16 : 256, chunk / 16 * 16));- eg_cap = (size_t) chunk;- tap_cap = std::max(kWin, 128);- } else // (only as asked for: --prefill-chunk 0, or --tp 1 with --threads off)- std::fprintf(stderr,- "ds4: prompts run in windows of %d tokens (%s): long prompts prefill far slower than in\n"- " chunks (a 7-GPU RTX 3090 rig, Maya-S read in place: 8K / 32K tokens 115 / 99 tokens/s,\n"- " 461 / 1010 in chunks of %d) - %s\n",- kWin, o.prefill_chunk == 0 ? "--prefill-chunk 0" : "--tp 1 with --threads off", kVlRows,- o.prefill_chunk == 0 ? "leave --prefill-chunk out for chunks"- : "leave --tp 1 out, or pass --threads on, for chunks");- // the vision tower on the first GPU, before the split: the layers get what it leaves- if (o.vision && !load_vision(err)) { // (a tower that cannot load leaves the model reading text: said loudly,- if (o.vision_only) return false; // and the server is told pictures are off - info_lines)- std::fprintf(stderr, "ds4: WARNING: the vision tower did not load (%s): pictures are OFF, the model reads "- "text only\n", err.c_str());- vision_err = err.empty() ? "unknown reason" : err;- err.clear();- vt.release();- img_tok = -1;- }- if (o.vision_only) return true;-- // layers: roles and the sources they read- if (!o.dspark) g.n_mtp = 0;- L.resize((size_t) (g.nl + g.n_mtp));- int last_src = -1;- for (int il = 0; il < g.nl + g.n_mtp; ++il) {- Layer& l = L[(size_t) il];- l.il = il;- l.mtp = il >= g.nl;- l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";- l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;- l.k_exp = l.mtp ? g.d_k : g.k_exp;- l.ratio = g.ratios[(size_t) il];- if (l.mtp) { // window attention only: no compressor, indexer or Engram- if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {- err = l.name + ": a DSpark block of compress ratio " + std::to_string(l.ratio) + ", " +- std::to_string(l.n_exp) + " experts / " + std::to_string(l.k_exp) + " is not covered";- return false;- }- if (!expert_layout(l, err)) return false;- continue;- }- l.kv_src = std::find(g.kv_src.begin(), g.kv_src.end(), il) != g.kv_src.end();- l.ix_src = std::find(g.ix_src.begin(), g.ix_src.end(), il) != g.ix_src.end();- l.owns_k = l.kv_src;- l.cand_src = il == g.cand_src;- l.uses_cand = g.cand_src >= 0 && g.cand_src < il && l.ix_src;- if (l.kv_src) {- Src s;- s.layer = il;- s.ratio = l.ratio;- s.cap = (o.max_context + l.ratio - 1) / l.ratio + 1;- srcs.push_back(s);- last_src = (int) srcs.size() - 1;- }- if (l.ratio > 0) {- if (last_src < 0 || srcs[(size_t) last_src].ratio != l.ratio) {- err = "layer " + std::to_string(il) + ": compress ratio " + std::to_string(l.ratio) +- " without a KV source of that ratio before it";- return false;- }- l.src = last_src;- l.ik_src = last_src;- }- auto eg = std::find(g.eg_layers.begin(), g.eg_layers.end(), il);- if (eg != g.eg_layers.end()) l.eg = (int) (eg - g.eg_layers.begin());- if (!expert_layout(l, err)) return false;- }-- // the split: each layer's dense weights + its experts, the head on the last GPU- for (Layer& l : L) {- const std::string& P = l.name;- size_t b = 0;- if (l.mtp && l.il == g.nl) b += mat_bytes("mtp.0.main_proj");- for (const char* m : {"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "ffn.shared_experts.w1",- "ffn.shared_experts.w2", "ffn.shared_experts.w3", "attn.compressor.wkv",- "attn.compressor.wgate", "attn.indexer.wq_b", "attn.indexer.wk", "engram.wkv"})- b += mat_bytes(P + m);- for (int gi = 0; gi < g.o_groups; ++gi) b += mat_bytes(P + "attn.wo_a.slice." + std::to_string(gi));- b += tensor_bytes(P + "hc_attn_fn") + tensor_bytes(P + "hc_ffn_fn") + tensor_bytes(P + "ffn.gate.weight");- b += (size_t) g.win * win_rb() + (2u << 20);- l.dense_bytes = b;- }- const size_t head_b = mat_bytes("head") + (size_t) g.vocab * 4 + (16u << 20);- const size_t per_dev_fixed = (size_t) o.reserve_mib << 20;- const size_t kv_per_src_dev = 0; // counted after the split- (void) kv_per_src_dev;- std::vector<double> avail(groups.size(), 0.0);- double avail_sum = 0;- for (size_t gi = 0; gi < groups.size(); ++gi) {- for (int m : groups[gi].mem)- avail[gi] += std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed - 400.0 * (1 << 20));- avail_sum += avail[gi];- }- double need_sum = (double) head_b;- for (const Layer& l : L) need_sum += (double) l.dense_bytes + (double) l.blob * l.n_exp;- {- // contiguous layer ranges, each group's share of the total by its free memory- size_t d = 0;- double cum = 0, target = avail[0] / avail_sum * need_sum;- std::vector<Layer*> order; // the drafter's blocks first (the first GPU), then the trunk's layers- for (Layer& l : L)- if (l.mtp) order.push_back(&l);- for (Layer& l : L)- if (!l.mtp) order.push_back(&l);- for (Layer* lp : order) {- Layer& l = *lp;- const double lb = (double) l.dense_bytes + (double) l.blob * l.n_exp;- if (cum + lb / 2 > target && d + 1 < groups.size()) {- ++d;- target += avail[d] / avail_sum * need_sum;- }- l.grp = (int) d;- cum += lb;- }- // with the chunked prompt path: the contiguous split that streams the least into its busiest member (its- // VRAM short of its layers' experts, per member) - a chunk's groups run as a pipeline, its slowest stage- // sets the pace; a token's decode passes every layer either way, the experts in VRAM the same in all- if (chunk > 0 && groups.size() > 1 && groups.size() <= 6 && std::getenv("DS4_SPLIT") == nullptr) {- const int G = (int) groups.size(), NL = g.nl;- std::vector<double> cap(groups.size());- for (int gi = 0; gi < G; ++gi) {- const int nm = (int) groups[(size_t) gi].mem.size();- double mn = 1e300;- for (int m : groups[(size_t) gi].mem)- mn = std::min(mn, std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed -- 400.0 * (1 << 20) - 1.1 * (1 << 30))); // (the chunk's buffers)- cap[(size_t) gi] = mn * nm;- }- for (const Layer& l : L)- if (l.mtp) cap[0] -= (double) l.dense_bytes + (double) l.blob * l.n_exp;- cap[(size_t) G - 1] -= (double) head_b;- std::vector<double> pe(NL + 1, 0.0), pd(NL + 1, 0.0); // prefix sums: experts, dense- for (int il = 0; il < NL; ++il) {- pe[(size_t) il + 1] = pe[(size_t) il] + (double) L[(size_t) il].blob * L[(size_t) il].n_exp;- pd[(size_t) il + 1] = pd[(size_t) il] + (double) L[(size_t) il].dense_bytes;- }- auto cost = [&](int gi, int a, int b) { // per-member streaming of layers [a, b) on group gi- const int nm = (int) groups[(size_t) gi].mem.size();- const double dense = (pd[(size_t) b] - pd[(size_t) a]) * (nm > 1 ? 1.3 : 1.0);- return std::max(0.0, pe[(size_t) b] - pe[(size_t) a] - (cap[(size_t) gi] - dense)) / nm;- };- // best[gi][b]: the least max cost of layers [0, b) over groups 0..gi (each at least one layer)- std::vector<std::vector<double>> best((size_t) G, std::vector<double>((size_t) NL + 1, 1e300));- std::vector<std::vector<int>> cut((size_t) G, std::vector<int>((size_t) NL + 1, 0));- for (int b = 1; b <= NL; ++b) best[0][(size_t) b] = cost(0, 0, b);- for (int gi = 1; gi < G; ++gi)- for (int b = gi + 1; b <= NL; ++b)- for (int a = gi; a < b; ++a) {- const double v = std::max(best[(size_t) gi - 1][(size_t) a], cost(gi, a, b));- if (v < best[(size_t) gi][(size_t) b]) {- best[(size_t) gi][(size_t) b] = v;- cut[(size_t) gi][(size_t) b] = a;- }- }- int b = NL;- for (int gi = G - 1; gi >= 0; --gi) {- const int a = gi > 0 ? cut[(size_t) gi][(size_t) b] : 0;- for (int il = a; il < b; ++il) L[(size_t) il].grp = gi;- b = a;- }- }- // DS4_SPLIT=a,b,..: the trunk layers each group takes (the drafter stays with the first)- if (const char* sp = std::getenv("DS4_SPLIT")) {- std::vector<int> n;- for (const char* q = sp; *q;) {- n.push_back(std::atoi(q));- while (*q && *q != ',') ++q;- if (*q == ',') ++q;- }- int il = 0;- for (size_t gi = 0; gi < n.size() && gi < groups.size(); ++gi)- for (int j = 0; j < n[gi] && il < g.nl; ++j) L[(size_t) il++].grp = (int) gi;- for (; il < g.nl; ++il) L[(size_t) il].grp = (int) groups.size() - 1;- }- for (Layer& l : L) {- l.dev = groups[(size_t) l.grp].mem[0];- l.nm = l.mtp ? 1 : (int) groups[(size_t) l.grp].mem.size(); // (the drafter's blocks: on the lead alone)- }- head_dev = L[(size_t) g.nl - 1].dev;- d_dev = g.n_mtp > 0 ? L[(size_t) g.nl].dev : 0;- }- // the groups' members: copies of each layer's device state, a member's part of an expert- for (Layer& l : L) {- l.ff0 = 0;- l.ffn = g.ff;- l.ah0 = l.og0 = 0;- l.ahn = g.heads;- l.ogn = g.o_groups;- if (l.nm == 1) continue;- const int na = head_members(l.nm);- if (g.heads % na != 0 || g.o_groups % na != 0 || g.ff % 128 != 0 || g.ff / 128 < l.nm) {- err = "tensor parallel " + std::to_string(l.nm) + ": the heads, output groups or FFN do not split";- return false;- }- for (int m = 1; m < l.nm; ++m) {- Layer c = l;- c.mates.clear();- c.mi = m;- c.dev = groups[(size_t) l.grp].mem[(size_t) m];- l.mates.push_back(std::move(c));- }- // each member's FFN columns and its part of an expert's layout: an EXL3 expert's in whole 128-blocks, as even- // as they go (split128, the odd blocks going round the members layer by layer); a GGUF's in even shares (its- // groups are only as wide as its experts' blocks split evenly: mx_blob_to_half's n_ff / nm)- const bool gguf_x = l.lay.fmt == gg::kFmtMXFP4;- for (int m = 0; m < l.nm; ++m) {- Layer& x = member(l, m);- const int ha = m - (l.nm - na); // (its place among the members with heads)- x.ahn = ha < 0 ? 0 : g.heads / na;- x.ah0 = ha < 0 ? 0 : ha * x.ahn;- x.ogn = ha < 0 ? 0 : g.o_groups / na;- x.og0 = ha < 0 ? 0 : ha * x.ogn;- if (gguf_x) {- x.ffn = g.ff / l.nm;- x.ff0 = m * x.ffn;- } else- split128(g.ff, l.nm, m, x.ff0, x.ffn, l.il % l.nm); // (rot: the odd blocks go round the members)- size_t off = 0;- for (int r = 0; r < 3; ++r) {- const int k = r < 2 ? g.dim : x.ffn, n = r < 2 ? x.ffn : g.dim;- x.hlay.K[r] = l.lay.K[r];- x.hlay.trellis[r] = off;- if (gguf_x) { // gate / up rows, down inputs: [n][k / block] blocks- const int ty = gg::expert_type(l.lay, r);- if (k % gg::type_block(ty) != 0) {- err = "tensor parallel " + std::to_string(l.nm) + ": the experts' " + gg::type_name(ty) +- " blocks do not split";- return false;- }- x.hlay.suh[r] = x.hlay.svh[r] = off;- off += align256((size_t) n * gg::type_row_bytes(ty, k));- continue;- }- off += align256(x3::trellis_bytes(k, n, l.lay.K[r]));- x.hlay.suh[r] = off;- off += align256((size_t) k * 2);- x.hlay.svh[r] = off;- off += align256((size_t) n * 2);- }- x.hlay.cb = l.lay.cb;- x.hlay.fmt = l.lay.fmt;- x.hblob = off;- }- }- // which devices hold each source's compressed KV / index keys- for (Src& s : srcs) {- s.ckv.assign(devs.size(), nullptr);- s.ik.assign(devs.size(), nullptr);- }- for (const Layer& l0 : L)- for (int m = 0; m < l0.nm; ++m) {- const Layer& l = member(l0, m);- if (l.src >= 0) {- srcs[(size_t) l.src].ckv[(size_t) l.dev] = (uint8_t*) 1;- if (l0.grp != L[(size_t) srcs[(size_t) l.src].layer].grp) srcs[(size_t) l.src].remote = true;- }- if (l.ix_src && l.ik_src >= 0) {- srcs[(size_t) l.ik_src].ik[(size_t) l.dev] = (uint8_t*) 1;- if (l0.grp != L[(size_t) srcs[(size_t) l.ik_src].layer].grp) srcs[(size_t) l.ik_src].remote = true;- }- }-- // per device: the scratch, the caches, the dense weights- for (size_t di = 0; di < devs.size(); ++di) {- Dev& d = devs[di];- const int D = g.dim, H = g.hc;- const size_t W = kWin, P = (size_t) kWin * kPicks, cap = (size_t) o.max_context + 64;- bool ok = dalloc(d, d.h, W * H * D, err) && dalloc(d, d.h2, W * H * D, err) && dalloc(d, d.x, W * D, err) &&- dalloc(d, d.qr, W * g.q_lora, err) && dalloc(d, d.q, W * g.heads * g.hd, err) &&- dalloc(d, d.kvr, W * g.hd, err) && dalloc(d, d.ckr, W * g.hd, err) &&- dalloc(d, d.cgr, W * g.hd, err) && dalloc(d, d.latent, W * g.hd, err) &&- dalloc(d, d.ikr, W * g.ix_hd, err) && dalloc(d, d.iq, W * g.ix_heads * g.ix_hd, err) &&- dalloc(d, d.iw, W * 64, err) && dalloc(d, d.scores, W * cap, err) &&- dalloc(d, d.bm, W * bm_stride(), err) && dalloc(d, d.o, W * g.heads * g.hd, err) &&- dalloc(d, d.oa, W * g.o_groups * g.o_lora, err) && dalloc(d, d.ao, W * D, err) &&- dalloc(d, d.mix, W * 32, err) && dalloc(d, d.a_pre, W * 8, err) && dalloc(d, d.a_post, W * 8, err) &&- dalloc(d, d.a_comb, W * 64, err) && dalloc(d, d.f_pre, W * 8, err) &&- dalloc(d, d.f_post, W * 8, err) && dalloc(d, d.f_comb, W * 64, err) && dalloc(d, d.pre, W * 8, err) &&- dalloc(d, d.glog, W * 1024, err) && dalloc(d, d.glog_a, 2 * W * 1024, err) &&- dalloc(d, d.rw, W * kPicks, err) &&- dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.ridx_a, 2 * kPicks, err) && dalloc(d, d.rw_a, 2 * kPicks, err) &&- dalloc(d, d.moe_out, W * D, err) &&- dalloc(d, d.sg, W * g.ff, err) && dalloc(d, d.su, W * g.ff, err) && dalloc(d, d.sh, W * g.ff, err) &&- dalloc(d, d.sh_out, W * D, err) && dalloc(d, d.cpu_part, W * D, err) &&- dalloc(d, d.freqs_win, 64, err) && dalloc(d, d.freqs_comp, 64, err) &&- dalloc(d, d.topk, W * topk_stride(), err) && dalloc(d, d.cand, W * cand_stride(), err) &&- dalloc(d, d.tok, 16, err) && dalloc(d, d.emb_rows, W * D, err) &&- dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&- dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&- dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);- if (ok && std::getenv("DS4_DBG_DEVH"))- std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);- if (!ok) return false;- d.attn_scr.max_t = kAttnScratchT;- d.attn_scr.cap = (int) topk_stride();- if (!dalloc(d, d.attn_scr.rows, (size_t) kAttnScratchT * topk_stride() * g.hd, err)) return false;- if (!g.eg_layers.empty()) {- const int cols = (pk.eg_ngram - 1) * pk.eg_heads;- ok = dalloc(d, d.eg_rows, W * cols * (pk.eg_hd + pk.eg_hd / 32), err) &&- dalloc(d, d.eg_x, W * cols * pk.eg_hd, err) && dalloc(d, d.eg_kv, W * (g.hc + 1) * D, err);- if (!ok) return false;- }- if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;- if (!halloc(d.bounce, bounce_bytes(), err)) return false;- if (!dalloc(d, d.xcnt, 4, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||- !dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||- !dalloc(d, d.n_groups, 4, err))- return false;- if (g.n_mtp > 0 && !dalloc(d, d.tap, (size_t) tap_cap * D, err)) return false;- if (g.n_mtp > 0 && (int) di == head_dev &&- (!dalloc(d, d_confv, kWin, err) || !dalloc(d, d_ints, 64, err) || !dalloc(d, d_specq, kWin, err)))- return false;- CK(cudaEventCreateWithFlags(&d.ev_out, cudaEventDisableTiming));- cudaSetDevice(d.id);- CK(cudaMemcpy(d.freqs_win, freqs_win.data(), freqs_win.size() * 4, cudaMemcpyHostToDevice));- CK(cudaMemcpy(d.freqs_comp, freqs_comp.data(), freqs_comp.size() * 4, cudaMemcpyHostToDevice));- for (Src& s : srcs) {- if (s.h_ckv == nullptr) { // (its events belong to the source layer's device: it records them)- if (!halloc(s.h_ckv, (size_t) kWin * ckv_rb(), err) || !halloc(s.h_ik, (size_t) kWin * ik_rb(), err))- return false;- CK(cudaSetDevice(devs[(size_t) L[(size_t) s.layer].dev].id));- CK(cudaEventCreateWithFlags(&s.ev_ckv, cudaEventDisableTiming));- CK(cudaEventCreateWithFlags(&s.ev_ik, cudaEventDisableTiming));- CK(cudaSetDevice(d.id));- }- if (s.ckv[di] != nullptr && !dalloc(d, s.ckv[di], (size_t) s.cap * ckv_rb(), err, &d.caches))- return false;- if (s.ik[di] != nullptr && !dalloc(d, s.ik[di], (size_t) s.cap * ik_rb(), err, &d.caches))- return false;- }- }- for (Src& sr : srcs) {- if (sr.remote && (!hmap(sr.x_stage, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag, 32, err)))- return false;- if (sr.remote && chunk > 0 &&- (!hmap(sr.x_stage1, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag1, 32, err) ||- !hmap(sr.x_ack, (size_t) 2 * devs.size() * 32, err)))- return false;- const int sg = L[(size_t) sr.layer].grp;- for (size_t di = 0; di < devs.size(); ++di)- if ((sr.ckv[di] != nullptr || sr.ik[di] != nullptr) && devs[di].grp != sg) sr.readers.push_back((int) di);- }- for (Group& G : groups) {- G.ar.nm = (int) G.mem.size();- G.ar.slots = 4;- G.ar.stride = (size_t) kWin * g.dim;- if (G.ar.nm > 1 && (!hmap(G.ar.data, (size_t) G.ar.nm * G.ar.slots * G.ar.stride, err) ||- !hmap(G.ar.flags, (size_t) G.ar.nm * G.ar.slots * 32, err)))- return false;- // three members or more: the reduce-scatter's sums and its per-block flags (allreduce)- if (G.ar.nm > 2 && (!hmap(G.ar.sums, (size_t) G.ar.slots * G.ar.stride, err) ||- !hmap(G.ar.bflags, (size_t) 2 * G.ar.nm * G.ar.slots * kArBlocks * 8, err)))- return false;- if (G.ar.nm > 1 && &G == &groups[(size_t) devs[(size_t) head_dev].grp] &&- (!hmap(G.lg_stage, (size_t) 2 * (G.ar.nm - 1) * kWin * ((g.vocab / 128 + G.ar.nm - 1) / G.ar.nm * 128), err) ||- !hmap(G.lg_flag, (size_t) 2 * (G.ar.nm - 1) * 32, err) || !hmap(G.lg_ack, (size_t) 2 * 32, err)))- return false;- G.arb = G.ar; // (a chunk's: two slots of chunk rows)- G.arb.slots = 2;- G.arb.stride = (size_t) chunk * g.dim;- if (chunk > 0 && G.arb.nm > 1 && (!hmap(G.arb.data, (size_t) G.arb.nm * G.arb.slots * G.arb.stride, err) ||- !hmap(G.arb.flags, (size_t) G.arb.nm * G.arb.slots * 32, err)))- return false;- if (chunk > 0 && G.arb.nm > 2 &&- (!hmap(G.arb.sums, (size_t) G.arb.slots * G.arb.stride, err) ||- !hmap(G.arb.bflags, (size_t) 2 * G.arb.nm * G.arb.slots * kArBlocks * 8, err)))- return false;- }- if (g.n_mtp > 0) {- if (!halloc(h_taps, (size_t) tap_cap * g.d_targets.size() * g.dim, err)) return false;- CK(cudaSetDevice(devs[(size_t) d_dev].id));- CK(cudaEventCreateWithFlags(&ev_seed, cudaEventDisableTiming));- CK(cudaEventRecord(ev_seed, devs[(size_t) d_dev].st));- }- // the dense weights, each device's layers by a thread of its own- {- std::vector<std::string> errs(devs.size());- std::vector<std::thread> th;- for (size_t di = 0; di < devs.size(); ++di)- th.emplace_back([this, di, &errs] {- std::string e;- if (!load_dense((int) di, e)) errs[di] = e;- });- for (auto& t : th) t.join();- for (auto& e : errs)- if (!e.empty()) {- err = e;- return false;- }- }- if (o.verbose)- std::fprintf(stderr, "ds4: dense weights loaded in %.1f s\n", (now_ms() - t0) / 1000.0);- // the markov steps over a head group of two or more (DS4_MK_SPLIT=0: the lead's whole rows, as before): E- // host-mapped, the steps' slots- // (only where every member holds its rows of the markov head: a type load_dense does not take keeps the lead's)- bool mk_rows = g.n_mtp > 0 && head_nm() > 1;- for (int m : groups[(size_t) devs[(size_t) head_dev].grp].mem) mk_rows = mk_rows && devs[(size_t) m].mk_share != nullptr;- if (mk_rows && !(std::getenv("DS4_MK_SPLIT") != nullptr && std::atoi(std::getenv("DS4_MK_SPLIT")) == 0)) {- const Tensor* te = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.embed.weight");- if (te == nullptr || te->dtype != "BF16" || te->bytes != (uint64_t) g.vocab * g.d_rank * 2) {- err = "the markov embed: not a BF16 [vocab][rank] table";- return false;- }- if (!hmap(h_mk_embed, (size_t) g.vocab * g.d_rank, err) || !pk.read(*te, h_mk_embed) ||- !hmap(mk_slots, (size_t) kWin * head_nm() * 4, err)) {- if (err.empty()) err = "the markov embed: read failed";- return false;- }- devs[(size_t) head_dev].mk_ints = d_ints; // (the lead's drafted tokens: where the confidence head reads them)- mk_split = true;- }-- // the embedding table, pinned on the host- {- const Tensor* t = pk.get("embed.weight");- const size_t nv = (size_t) g.vocab * g.dim;- const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;- // other block types (a GGUF whose dense tensors are Q4_K / Q5_K / Q6_K: the IST-DAS sets' Q5_K embed.weight):- // each row through ggml's to_float, to the same BF16 the host table holds- const int ety = t != nullptr ? gg_type_of(t->dtype) : -1;- const size_t erb = ety > 0 ? gg::type_row_bytes(ety, g.dim) : 0;- const bool qrow = t != nullptr && !(t->dtype == "BF16") && erb > 0 && t->bytes == (size_t) g.vocab * erb;- if (t == nullptr || (!q8 && !qrow && (t->dtype != "BF16" || t->bytes != nv * 2))) {- err = "embed.weight: not a BF16 (or Q8_0 / a ggml block type) [vocab][dim] table";- return false;- }- if (!halloc(embed_pinned, nv, err)) return false;- if (qrow) { // (ggml's decode of each row, rounded to BF16 as the Q8_0 path rounds)- std::vector<uint8_t> row(erb);- for (size_t r = 0; r < (size_t) g.vocab; ++r) {- if (!pk.read_at(t->file, t->off + r * erb, erb, row.data())) {- err = "embed.weight: read failed";- return false;- }- if (!gg::gg_row_to_bf16(ety, row.data(), g.dim, embed_pinned + r * g.dim)) {- err = "embed.weight: ggml has no decode for " + t->dtype;- return false;- }- }- } else if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)- std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);- for (size_t r = 0; r < (size_t) g.vocab; ++r) {- if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {- err = "embed.weight: read failed";- return false;- }- const std::vector<float> v = q8_values(row.data(), (size_t) g.dim);- for (size_t c = 0; c < (size_t) g.dim; ++c) {- uint32_t u;- std::memcpy(&u, &v[c], 4);- u += 0x7fffu + ((u >> 16) & 1u); // round to nearest even- embed_pinned[r * g.dim + c] = (uint16_t) (u >> 16);- }- }- } else if (!pk.read(*t, embed_pinned)) {- err = "embed.weight: read failed";- return false;- }- if (std::getenv("DS4_DBG_EMB")) std::fprintf(stderr,- "DBG emb pinned: %04x %04x %04x %04x (row1: %04x %04x)\n",- (unsigned) embed_pinned[0], (unsigned) embed_pinned[1],- (unsigned) embed_pinned[2], (unsigned) embed_pinned[3],- (unsigned) embed_pinned[(size_t) g.dim], (unsigned) embed_pinned[(size_t) g.dim + 1]);-- }- // the host side of the CPU lane and the Engram rows- const int cols = (pk.eg_ngram - 1) * pk.eg_heads;- if (!halloc(h_x, (size_t) kWin * g.dim, err) || !halloc(h_w, (size_t) kWin * kPicks, err) ||- !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) || !halloc(h_idx_a, (size_t) 2 * kPicks, err) ||- !halloc(h_emb, (size_t) std::max(kWin, chunk) * g.dim, err) ||- !halloc(h_eg, (size_t) std::max(1, pk.eg_layers) * eg_cap * cols * (pk.eg_hd + pk.eg_hd / 32) + 64, err))- return false;- if (pk.eg_layers > 0) {- // the Engram rows' reader: threads enough to keep an SSD's queue busy (reads from the page cache need few);- // the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)- const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;- const unsigned hw = std::max(1u, std::thread::hardware_concurrency());- int nth = (int) std::min(16u, std::max(4u, hw / 2));- if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));- const uint64_t avail = ram_avail();- uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));- if (const char* e = std::getenv("DS4_ENGRAM_CACHE_MB")) mb = std::strtoull(e, nullptr, 10);- eg_cache.init((size_t) ((mb << 20) / row), row);- std::vector<std::string> paths;- for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);- eg_io.start(paths, nth);- for (int li = 0; li < pk.eg_layers; ++li) {- eg_win.emplace_back(new EgBatch());- const std::string id = std::to_string(g.eg_layers[(size_t) li]);- const Tensor* tw = pk.get("layers." + id + ".engram.embed.weight");- // The row layout the reader and the dequant take. A type not here (or no tensor at all) would be read- // as some other format's bytes - the old code then read the scale tensor of a pack that has none and- // crashed in eg_issue - so refuse the pack once, with the type's name, before any token is read.- int fmt;- if (tw == nullptr) {- std::fprintf(stderr, "ds4: Engram layer %s: engram.embed.weight is missing from the pack\n", id.c_str());- std::exit(2);- } else if (tw->dtype == "MXFP8") {- fmt = 0;- } else if (tw->dtype == "Q6_K") {- fmt = 1;- } else if (tw->dtype == "IQ4_NL") {- fmt = 2;- } else if (pk.get("layers." + id + ".engram.embed.scale") != nullptr) {- fmt = 3; // (an old pack: FP8 codes in one tensor, their E8M0 scales in another)- } else {- std::fprintf(stderr,- "ds4: Engram layer %s: engram.embed.weight type %s is not one the Engram reader handles "- "(MXFP8, Q6_K, IQ4_NL)\n",- id.c_str(), tw->dtype.c_str());- std::exit(2);- }- if (fmt == 1 && pk.eg_hd % 256 != 0) {- err = "Engram layer " + id + ": Q6_K rows of " + std::to_string(pk.eg_hd) +- " values (not whole superblocks of 256)";- return false;- }- if (fmt == 2 && pk.eg_hd % 32 != 0) {- err = "Engram layer " + id + ": IQ4_NL rows of " + std::to_string(pk.eg_hd) +- " values (not whole blocks of 32)";- return false;- }- eg_fmt.push_back((uint8_t) fmt);- }- if (o.verbose)- std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "- "their layer; row cache %llu MB (%zu rows)\n",- nth, (unsigned long long) mb, eg_cache.capacity());- }- prof.assign((size_t) (g.nl + g.n_mtp) * g.n_exp, 0u);- if (!o.profile.empty()) {- std::FILE* f = std::fopen(o.profile.c_str(), "rb");- if (f != nullptr) {- std::vector<uint32_t> p(prof.size());- const size_t n = std::fread(p.data(), 4, p.size(), f);- if (n == p.size() || n == (size_t) g.nl * g.n_exp) std::copy(p.begin(), p.begin() + (long) n, prof.begin());- std::fclose(f);- }- }-- // the chunked prompt path's buffers (before the experts: they take what is left)- if (chunk > 0 && !alloc_chunks(err)) return false;- // the CPU lane's experts read in place (decided above): how much of them the load reads ahead, from what VRAM- // now leaves over- if (file_experts) {- double room = 0, all = 0; // the VRAM place_experts will find for experts; every expert's bytes- for (Dev& d : devs) {- cudaSetDevice(d.id);- size_t fr = 0, tot = 0;- CK(cudaMemGetInfo(&fr, &tot));- room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;- }- for (const Layer& l : L) all += (double) l.blob * l.n_exp;- const double need = std::max(0.0, all - room);- const uint64_t avail = ram_avail();- warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;- if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;- if (store_gb() > 0) warm_bytes = 0; // (the store is the RAM tier: nothing to warm in the OS cache)- std::fprintf(stderr,- "ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "- "(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",- need / 1e9, o.mmap_experts > 0 ? " (--mmap-experts on)"- : pk.gguf ? " (--mmap-experts off copies them into RAM)" : ", not copied into RAM",- avail == ~uint64_t{0} ? 0.0 : avail / 1e9, warm_bytes / 1e9, chunk);- }- // the CPU lane: one pool of workers per NUMA node, each expert split in NH column parts (part X on node X); in- // place, one part (the files' layout) and one pool over every node's cores (the pages lie where the cache put them)- cores = node_cores();- NH = file_experts ? 1 : (int) std::min<size_t>(2, cores.size());- if (NH > 1 && ((g.ff / NH) % 128 != 0 || (g.dim / NH) % 128 != 0)) NH = 1;- if (!pk.gguf) make_pools(); // (a GGUF's after its experts: copy_experts decides NH)- {- const size_t E = (size_t) g.dim, FF = (size_t) g.ff, GQ = E / 256, DQ = FF / 128, S = kSlots;- c_xh.resize((size_t) NH * S * 2 * E);- c_xsb.resize((size_t) NH * S * 2 * (E / 128));- c_acc.resize((size_t) NH * S * 2 * GQ * (FF / NH));- c_xhd.resize(S * FF);- c_xsd.resize(S * (FF / 128));- c_accd.resize((size_t) NH * S * DQ * (E / NH));- c_yd.resize(S * E);- }- // the experts: VRAM slots in what each device has left, the rest on the host- if (!place_experts(err)) return false;- if (!load_experts(err)) return false;- if (pk.gguf) make_pools();- if (chunk > 0) {- // the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots- const double tr = now_ms();- for (auto& m : host_maps)- if (cudaHostRegister(m.first, m.second, cudaHostRegisterPortable) != cudaSuccess) {- err = "pinning the CPU lane's experts (" + std::to_string(m.second >> 20) + " MiB) failed";- return false;- }- if (o.verbose) std::fprintf(stderr, "ds4: CPU lane's experts pinned in %.1f s\n", (now_ms() - tr) / 1000.0);- if (file_experts) { // in place: each device's staging for its ring, and its readers (DS4_READERS, 8)- size_t mr = 0;- for (const Layer& l : L)- if (!l.mtp) {- for (uint64_t r : l.region_bytes) mr = std::max<size_t>(mr, r);- if (l.lay.fmt == gg::kFmtMXFP4) { // (a GGUF's: read as its blob)- mr = std::max(mr, l.blob);- for (int r = 0; r < 3; ++r) flow_part[r] = std::max(flow_part[r], mx_part_bytes(l, r));- }- }- const char* nr = std::getenv("DS4_READERS");- const int n_readers = std::max(1, nr != nullptr ? std::atoi(nr) : 8);- const bool flow_on = std::getenv("DS4_FLOW") == nullptr || std::atoi(std::getenv("DS4_FLOW")) != 0;- // (a tensor-parallel group's members on the flow only with DS4_FLOW_TP=1 for now: each reads the- // whole down of every expert it streams, 25 MB an expert for a pair, 18.8 alone - 3x V100, the- // 17-token prompt from the page cache: 2.5-2.7 s on the flow, 1.3-1.6 s in the group's windows)- const bool flow_tp = std::getenv("DS4_FLOW_TP") != nullptr && std::atoi(std::getenv("DS4_FLOW_TP")) != 0;- for (size_t di = 0; di < chunks.size(); ++di) {- auto& cp = chunks[di];- if (cp == nullptr || cp->ring == nullptr) continue;- cp->stage_stride = (mr + 4095) & ~(size_t) 4095;- // the flow: every trunk layer of the device a GGUF's (alone or a tensor-parallel group's member)- bool fl = flow_on;- int n_exp_max = 0;- for (const Layer& l : L)- if (!l.mtp)- for (int m = 0; m < l.nm; ++m)- if (member(l, m).dev == (int) di) {- // (a group's members: each its share of the down's blocks in every row)- fl = fl && l.lay.fmt == gg::kFmtMXFP4 &&- (l.nm == 1 || (flow_tp && (g.ff / l.nm) %- std::max(1, gg::type_block(gg::expert_type(l.lay, 2))) == 0));- n_exp_max = std::max(n_exp_max, l.n_exp);- }- if (fl) // (each part read unbuffered from its offset rounded down: a page more each)- cp->stage_stride = std::max(cp->stage_stride, flow_stride());- cp->S = cp->ring_n;- if (!halloc(cp->stage, (size_t) cp->S * cp->stage_stride, err)) return false;- cp->readers = std::make_unique<Pool>(n_readers - 1, std::vector<int>{}, 50);- if (!fl) continue;- cudaSetDevice(devs[di].id);- cp->flow = true;-#if defined(STRATA_METAL)- cp->direct = std::getenv("DS4_FLOW_DIRECT") == nullptr || std::atoi(std::getenv("DS4_FLOW_DIRECT")) != 0;-#endif- cp->dev = (int) di;- cp->io = std::make_unique<IoQueue>(n_readers);- cp->stage_item.assign((size_t) cp->S, -1);- cp->stage_ev.resize((size_t) cp->S);- cp->ring_fill.resize((size_t) cp->ring_n);- cp->ring_free.resize((size_t) cp->ring_n);- for (auto* v : {&cp->stage_ev, &cp->ring_fill, &cp->ring_free})- for (cudaEvent_t& e : *v) CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));- // (a region is written again only bc_n batches later - more than a layer's items, so after that- // layer's bounds wait, which every copy from it precedes)- cp->bc_n = n_exp_max + 2;- cp->bc_w = cp->ring_n + 1;- if (!halloc(cp->bc_pin, (size_t) cp->bc_n * cp->bc_w, err) ||- !dalloc(devs[di], cp->bc_dev, (size_t) cp->bc_n * cp->bc_w, err))- return false;- }- // every device with layers on the flow: a prompt reads only the experts it picks, once, so a short one- // takes the chunked path too (one pass, where windows of 16 read the CPU lane's picks again for each- // window, a page fault at a time) - DS4_CHUNK_MIN still overrides- bool all_flow = true;- for (const auto& cp : chunks) all_flow = all_flow && (cp == nullptr || cp->ring == nullptr || cp->flow);- if (all_flow && std::getenv("DS4_CHUNK_MIN") == nullptr) chunk_min = kFlowChunkMin;- }- for (Layer& l : L) {- if (l.mtp) continue;- for (int m = 0; m < l.nm; ++m) { // (each member its own: the flow's members decide them each)- Layer& x = member(l, m);- if (!halloc(x.vs_pin, (size_t) l.n_exp, err)) return false;- if (!dalloc(devs[(size_t) x.dev], x.vs_dev, (size_t) l.n_exp, err)) return false;- }- }- }- {- void* m = nullptr;- if (cudaHostAlloc(&m, sizeof(Req) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {- err = "host-mapped memory for the decode path's requests";- return false;- }- host_allocs.push_back(m);- reqs = (Req*) m;- if (cudaHostAlloc(&m, sizeof(Resp) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {- err = "host-mapped memory for the decode path's answers";- return false;- }- host_allocs.push_back(m);- resps = (Resp*) m;- std::memset((void*) reqs, 0, sizeof(Req) * kRing);- std::memset((void*) resps, 0, sizeof(Resp) * kRing);- for (auto& x : layer_of) x.store(-1);- ahead_fl.assign(L.size() * 1024, 0);- ahead_rank.assign(L.size() * 1024, 0);- ahead_for.assign(L.size(), 0);- const int core = !cores.empty() && !cores[0].empty() ? cores[0][0] : -1;- service = std::thread([this, core] {- if (core >= 0) pin_self(core);- serve_lane();- });- if (ahead_mode > 0) pf_th = std::thread([this] { prefetch_loop(); });- }- if (o.expert_cache) {- size_t maxb = 0;- for (const Layer& l : L) maxb = std::max(maxb, l.blob);- const int nt = std::max(1, std::min(c_inflight, std::getenv("DS4_CACHE_THREADS") ? std::atoi(std::getenv("DS4_CACHE_THREADS")) : 1));- sw_bufs.resize((size_t) nt);- for (SwapBufs& B : sw_bufs) {- if (!halloc(B.a, maxb, err) || !halloc(B.b, maxb, err) || !halloc(B.h, maxb, err)) return false;- for (Dev& d : devs) {- cudaSetDevice(d.id);- cudaStream_t cs = nullptr;- CK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));- B.st.push_back(cs);- }- }- for (SwapBufs& B : sw_bufs) cache_ths.emplace_back([this, &B] { cache_loop(B); });- }- for (Dev& d : devs) {- cudaSetDevice(d.id);- CK(cudaStreamSynchronize(d.st));- }- if (par || chunk > 0) // (the windows' threads; the prompt path's, whatever mode the windows take)- for (size_t di = 0; di < devs.size(); ++di) {- workers.push_back(std::make_unique<Worker>());- workers.back()->start();- }- calibrate_paths();- init_store(); // (last: the store's arena is registered with the driver - page-locked memory the other pinned allocations must come before)- preload_store();- if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());- reset();- return true;- }--- // the caches' row bytes, packed as DeepSeek keeps them (ds4_kernels.cuh): the window ring's (FP8), the compressed- // KV's (FP4 + E4M3 scales), the index keys' (FP4 + power-of-two scales)- size_t win_rb() const { return kv_row_bytes(kKvFp8, g.hd); }- size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }- size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }-- // ---- the chunked prompt path- size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows- size_t chunk_hand_bytes() const {- return (size_t) chunk * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);- }-- bool alloc_chunks(std::string& err) {- chunks.resize(devs.size());- const int D = g.dim, H = g.hc, HD = g.hd;- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;- for (size_t di = 0; di < devs.size(); ++di) {- Dev& d = devs[di];- cudaSetDevice(d.id);- chunks[di] = std::make_unique<Chunk>();- Chunk& c = *chunks[di];- const int nm = (int) groups[(size_t) d.grp].mem.size();- const int NHL = g.heads / head_members(nm), HQ = NHL * HD, OG = g.o_groups / head_members(nm);- int FF = 0, ff0_ = 0;- split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)- const size_t C = (size_t) chunk;- c.C = chunk;- c.A = std::max(64, (int) (((size_t) 8 << 20) / (size_t) HQ) / 64 * 64);- // (the indexer's slab: its scores rows are the context wide - about 64 MB of them)- c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));- c.w16_elems = (size_t) 8 << 20;- bool has_eg = false, has_first = false;- size_t stride = 0;- for (const Layer& l : L)- for (int m = 0; m < l.nm; ++m)- if (member(l, m).dev == (int) di && !l.mtp) {- has_eg = has_eg || l.eg >= 0;- has_first = has_first || l.il == 0;- stride = std::max(stride, align256(unit(member(l, m))));- }- const size_t A = (size_t) c.A, AI = (size_t) c.AI, ss = (size_t) o.max_context + 64;- bool ok = dalloc(d, c.h, C * H * D, err) && dalloc(d, c.x, C * D, err) && dalloc(d, c.acc, C * D, err) &&- dalloc(d, c.x16, C * D, err) && dalloc(d, c.qr, C * g.q_lora, err) &&- dalloc(d, c.qr16, C * g.q_lora, err) && dalloc(d, c.kvr, C * HD, err) &&- dalloc(d, c.ckr, C * HD, err) && dalloc(d, c.cgr, C * HD, err) &&- dalloc(d, c.latent, (C + 2) * HD, err) && dalloc(d, c.lat16, (C + 2) * HD, err) &&- dalloc(d, c.ikr, (C + 2) * g.ix_hd, err) && dalloc(d, c.ext, ((size_t) g.win + C) * HD, err) &&- dalloc(d, c.q, A * HQ, err) && dalloc(d, c.o, A * HQ, err) && dalloc(d, c.o16, A * HQ, err) &&- dalloc(d, c.oa, A * OG * g.o_lora, err) && dalloc(d, c.oa16, A * OG * g.o_lora, err) &&- dalloc(d, c.iq, AI * g.ix_heads * g.ix_hd, err) && dalloc(d, c.iw, C * 64, err) &&- dalloc(d, c.scores, AI * ss, err) && dalloc(d, c.bm, AI * bm_stride(), err) &&- dalloc(d, c.topk, C * topk_stride(), err) && dalloc(d, c.cand, C * cand_stride(), err) &&- dalloc(d, c.mix, C * 32, err) && dalloc(d, c.a_pre, C * 8, err) && dalloc(d, c.a_post, C * 8, err) &&- dalloc(d, c.a_comb, C * 64, err) && dalloc(d, c.f_pre, C * 8, err) &&- dalloc(d, c.f_post, C * 8, err) && dalloc(d, c.f_comb, C * 64, err) && dalloc(d, c.pre, C * 8, err) &&- dalloc(d, c.glog, C * 1024, err) && dalloc(d, c.ridx, C * kPicks, err) &&- dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&- dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&- dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&- dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&- halloc(c.bounds_pin, 2048, err);- if (!ok) return false;- if (has_first && !dalloc(d, c.emb, C * D, err)) return false;- if (has_eg) {- const size_t ES = 256, per = (size_t) cols * (ehd + ehd / 32);- if (!dalloc(d, c.eg_x, ES * cols * ehd, err) || !dalloc(d, c.eg_x16, ES * cols * ehd, err) ||- !dalloc(d, c.eg_kv, ES * (H + 1) * D, err) || !dalloc(d, c.eg_rows, C * per, err))- return false;- }- if (stride > 0) { // DS4_RING_MB (400) of slots in kParts batches- const size_t mb = std::getenv("DS4_RING_MB") ? (size_t) std::atoi(std::getenv("DS4_RING_MB")) : 400;- c.ring_stride = stride;- c.ring_n = std::max(Chunk::kParts, (int) ((mb << 20) / stride) / Chunk::kParts * Chunk::kParts);- if (!dalloc(d, c.ring, (size_t) c.ring_n * stride, err)) return false;- }- CK(cudaStreamCreateWithFlags(&c.cs, cudaStreamNonBlocking));- for (int i = 0; i < Chunk::kParts; ++i) {- CK(cudaEventCreateWithFlags(&c.filled[i], cudaEventDisableTiming));- CK(cudaEventCreateWithFlags(&c.freed[i], cudaEventDisableTiming));- }- if (cublasCreate(&c.blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(c.blas, d.st) != CUBLAS_STATUS_SUCCESS) {- err = "cuBLAS on CUDA" + std::to_string(d.id);- return false;- }- for (int sl = 0; sl < 2; ++sl) {- if (d.mi == 0 && (!hmap(c.hand[sl], chunk_hand_bytes(), err) || !hmap(c.hand_flag[sl], 32, err))) return false;- if (has_first && !halloc(c.emb_host[sl], C * D, err)) return false;- if (has_eg && !halloc(c.eg_host[sl], C * pk.eg_layers * (size_t) cols * (ehd + ehd / 32), err)) return false;- CK(cudaEventCreateWithFlags(&c.emb_ev[sl], cudaEventDisableTiming));- CK(cudaEventCreateWithFlags(&c.eg_ev[sl], cudaEventDisableTiming));- }- if (d.mi == 0 && !hmap(c.hand_ack, (size_t) 2 * 8 * 32, err)) return false;- }- return true;- }-- // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, cuBLAS- bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,- std::string& err) {- if (rows <= 0) return true;- const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);- for (int c0 = 0; c0 < m.n; c0 += cols) {- const int nc = std::min(cols, m.n - c0);- if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)- else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);- const float one = 1.0f;- const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,- m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,- CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);- if (st != CUBLAS_STATUS_SUCCESS) {- err = "cublasGemmEx: status " + std::to_string((int) st);- return false;- }- }- return true;- }- // y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)- bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,- float* y, int ldy, std::string& err) {- const float zero = 0.0f;- const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, rows, k, &alpha, W, CUDA_R_16F, k,- x16, CUDA_R_16F, ldx, &zero, y, CUDA_R_32F, ldy, CUBLAS_COMPUTE_32F,- CUBLAS_GEMM_DEFAULT_TENSOR_OP);- if (st != CUBLAS_STATUS_SUCCESS) {- err = "cublasGemmEx (F16 weights): status " + std::to_string((int) st);- return false;- }- return true;- }- // hc_mix over a chunk: the dots by an F32 GEMM, then the rms scale- bool hc_mix_chunk(Dev& d, Chunk& c, const float* h, const float* fn, int T, float* mixes, std::string& err) {- const int hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;-#if defined(STRATA_METAL)- if (!std::getenv("DS4_CHUNK_SGEMM")) { // the window path's fused kernel: no F32 GEMM, no second pass- hc_mix(h, fn, hcd, n_mix, T, g.eps, mixes, d.st);- return true;- }-#endif- const float one = 1.0f, zero = 0.0f;- if (cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, fn, hcd, h, hcd, &zero, mixes, 32) !=- CUBLAS_STATUS_SUCCESS) {- err = "cublasSgemm (mHC mixes)";- return false;- }- hc_mix_scale(h, hcd, n_mix, T, g.eps, mixes, d.st);- return true;- }-- // expert e of l (on the CPU lane: its NH column parts) -> member m's VRAM copy at dst, in the GPU layout (the full- // blob, or a group member's part), by DMA from the pinned parts on the copy stream (in place: from `src`, its file- // region read into pinned staging)- bool stream_expert(const Layer& l, int m, int e, uint8_t* dst, cudaStream_t cs, std::string& err,- const uint8_t* src = nullptr) {- if (l.lay.fmt == gg::kFmtMXFP4) { // the blob, or member m's rows / blocks: from `src` (the expert read into- // pinned staging as its blob) or from the file's mapping- auto mat_r = [&](int r) { return src != nullptr ? src + l.lay.trellis[r] : mx_mat(l, e, r); };- bool ok = true;- if (l.nm == 1) {- if (src != nullptr)- ok = cudaMemcpyAsync(dst, src, l.blob, cudaMemcpyHostToDevice, cs) == cudaSuccess;- else- for (int r = 0; r < 3; ++r)- ok = ok && cudaMemcpyAsync(dst + l.lay.trellis[r], mat_r(r), mx_part_bytes(l, r),- cudaMemcpyHostToDevice, cs) == cudaSuccess;- } else {- const int ffh = g.ff / l.nm;- const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),- dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),- dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);- for (int r = 0; r < 2; ++r)- ok = ok && cudaMemcpyAsync(dst + l.hlay.trellis[r], mat_r(r) + (size_t) m * ffh * gr,- (size_t) ffh * gr, cudaMemcpyHostToDevice, cs) == cudaSuccess;- ok = ok && cudaMemcpy2DAsync(dst + l.hlay.trellis[2], dh, mat_r(2) + (size_t) m * dh, dr, dh,- (size_t) g.dim, cudaMemcpyHostToDevice, cs) == cudaSuccess;- }- if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";- return ok;- }- const auto& parts = l.hh[(size_t) e];- const int D = g.dim, FF = g.ff;- auto cp2 = [&](uint8_t* d0, size_t dp, const uint8_t* s0, size_t sp, size_t w, size_t h) {- return cudaMemcpy2DAsync(d0, dp, s0, sp, w, h, cudaMemcpyHostToDevice, cs) == cudaSuccess;- };- auto cp = [&](uint8_t* d0, const uint8_t* s0, size_t n) {- return cudaMemcpyAsync(d0, s0, n, cudaMemcpyHostToDevice, cs) == cudaSuccess;- };- bool ok = true;- if (NH == 1) { // one part: the whole expert in exllamav3's layout, its pieces at hl's offsets- const uint8_t* p = src != nullptr ? src : parts[0];- if (l.nm == 1) {- for (int r = 0; r < 3; ++r)- ok = ok && cp(dst + l.lay.trellis[r], p + l.hl.trellis[r], l.piece_bytes[r][0]) &&- cp(dst + l.lay.suh[r], p + l.hl.suh[r], (size_t) l.hl.k[r] * 2) &&- cp(dst + l.lay.svh[r], p + l.hl.svh[r], (size_t) l.hl.n[r] * 2);- } else { // member m's part (its FFN columns [f0, f0 + fn)), cut as blob_to_half cuts a blob- const Layer& x = member(l, m);- const int f0 = x.ff0, fn = x.ffn;- for (int r = 0; r < 2; ++r) {- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) FF / 16, nth = (size_t) fn / 16;- ok = ok && cp2(dst + x.hlay.trellis[r], nth * tile, p + l.hl.trellis[r] + (size_t) (f0 / 16) * tile,- nt * tile, nth * tile, (size_t) D / 16) &&- cp(dst + x.hlay.suh[r], p + l.hl.suh[r], (size_t) D * 2) &&- cp(dst + x.hlay.svh[r], p + l.hl.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);- }- const size_t row = (size_t) (D / 16) * 32 * l.lay.K[2];- ok = ok && cp(dst + x.hlay.trellis[2], p + l.hl.trellis[2] + (size_t) (f0 / 16) * row,- (size_t) (fn / 16) * row) &&- cp(dst + x.hlay.suh[2], p + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2) &&- cp(dst + x.hlay.svh[2], p + l.hl.svh[2], (size_t) D * 2);- }- } else if (l.nm == 1) { // the full blob: each matrix's columns from the two parts side by side- for (int r = 0; r < 3; ++r) {- const size_t tile = (size_t) 32 * l.lay.K[r];- const int k = l.hl.k[r], nh = l.hl.n[r]; // part: k x nh- for (int X = 0; X < 2; ++X) {- ok = ok && cp2(dst + l.lay.trellis[r] + (size_t) X * (nh / 16) * tile, (size_t) (2 * nh / 16) * tile,- parts[(size_t) X] + l.hl.trellis[r], (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile,- (size_t) (k / 16));- ok = ok && cp(dst + l.lay.svh[r] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[r], (size_t) nh * 2);- }- ok = ok && cp(dst + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);- }- } else { // member m (FFN columns [f0, f0 + fn)): gate / up from the parts holding them; down rows [f0, ..) of- // both parts' columns- const Layer& x = member(l, m);- const int f0 = x.ff0, fn = x.ffn;- for (int r = 0; r < 2; ++r) {- const size_t tile = (size_t) 32 * l.lay.K[r];- const int hn = l.hl.n[r]; // part X: columns [X hn, (X + 1) hn)- for (int X = 0; X < 2; ++X) {- const int a = std::max(f0, X * hn), b = std::min(f0 + fn, (X + 1) * hn);- if (a >= b) continue;- ok = ok && cp2(dst + x.hlay.trellis[r] + (size_t) ((a - f0) / 16) * tile, (size_t) (fn / 16) * tile,- parts[(size_t) X] + l.hl.trellis[r] + (size_t) ((a - X * hn) / 16) * tile,- (size_t) (hn / 16) * tile, (size_t) ((b - a) / 16) * tile, (size_t) (D / 16)) &&- cp(dst + x.hlay.svh[r] + (size_t) (a - f0) * 2, parts[(size_t) X] + l.hl.svh[r] + (size_t) (a - X * hn) * 2,- (size_t) (b - a) * 2);- }- ok = ok && cp(dst + x.hlay.suh[r], parts[0] + l.hl.suh[r], (size_t) D * 2);- }- const size_t tile = (size_t) 32 * l.lay.K[2];- const int nh = D / 2;- for (int X = 0; X < 2; ++X) {- ok = ok && cp2(dst + x.hlay.trellis[2] + (size_t) X * (nh / 16) * tile, (size_t) (D / 16) * tile,- parts[(size_t) X] + l.hl.trellis[2] + (size_t) (f0 / 16) * (nh / 16) * tile,- (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile, (size_t) (fn / 16));- ok = ok && cp(dst + x.hlay.svh[2] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[2], (size_t) nh * 2);- }- ok = ok && cp(dst + x.hlay.suh[2], parts[0] + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);- }- if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";- return ok;- }-- // a device's streamed experts: batches of ring_n / kParts of each of its layers' CPU-lane experts, chunk by chunk- // in layer order; the copies run at most kParts batches ahead of the compute- struct Batch {- int k = 0, il = 0, j0 = 0, n = 0; // chunk, layer, its host_list range- };- struct Streamer {- std::vector<Batch> b;- size_t cp = 0, kp = 0; // the next batch to copy, to compute- std::atomic<size_t> done{0}; // cp, for read_ahead's thread- };- // in place (file_experts): the batches' file regions read ahead of their copies - POSIX_FADV_WILLNEED, the kernel- // reading them in the background in large requests, where the copies alone fault the pages in a few at a time -- // at most DS4_READAHEAD_MB (2048) beyond the batch being copied; each region by its layer's first member alone- void read_ahead(const Streamer& S, int di, const std::atomic<bool>& stop) const {- static const double cap = (std::getenv("DS4_READAHEAD_MB") ? std::atof(std::getenv("DS4_READAHEAD_MB")) : 2048.0) *- 1048576.0;- std::deque<std::pair<size_t, double>> issued; // (batch, bytes) not yet copied- double ahead = 0;- for (size_t q = 0; q < S.b.size() && !stop.load(std::memory_order_relaxed);) {- const size_t done = S.done.load(std::memory_order_relaxed);- while (!issued.empty() && issued.front().first < done) {- ahead -= issued.front().second;- issued.pop_front();- }- if (ahead > cap) {- std::this_thread::sleep_for(std::chrono::microseconds(500));- continue;- }- const Batch& bt = S.b[q];- const Layer& l0 = L[(size_t) bt.il];- double nb = 0;- if (l0.dev == di)- for (int i = 0; i < bt.n; ++i) {- const int e = l0.host_list[(size_t) (bt.j0 + i)];- if (l0.lay.fmt == gg::kFmtMXFP4) {- if (copy_mode && l0.ram[(size_t) e] >= 0) continue; // (in RAM: nothing to read)- for (int r = 0; r < 3; ++r)- os::advise(pk.fds[(size_t) l0.mx_file[r]], (l0.mx_off[r] + (uint64_t) e * l0.mx_stride[r]), mx_part_bytes(l0, r), true);- nb += (double) l0.blob;- continue;- }- os::advise(pk.fds[(size_t) l0.region_file[(size_t) e]], l0.region_off[(size_t) e], l0.region_bytes[(size_t) e], true);- nb += (double) l0.region_bytes[(size_t) e];- }- issued.push_back({q++, nb});- ahead += nb;- }- }- bool copy_ahead(Dev& d, Chunk& c, Streamer& S, int di, std::string& err) {- const int P = Chunk::kParts;- while (S.cp < S.b.size() && S.cp < S.kp + P) {- const Batch& bt = S.b[S.cp];- const int half = (int) (S.cp % P);- const Layer& l0 = L[(size_t) bt.il];- int mi = 0;- for (int m = 0; m < l0.nm; ++m)- if (member(l0, m).dev == di) mi = m;- if (S.cp >= (size_t) P) CK(cudaStreamWaitEvent(c.cs, c.freed[half], 0));- uint8_t* sb = nullptr; // in place: the batch's regions in the part's staging slots- if (c.stage != nullptr) {- if (S.cp >= (size_t) P) CK(cudaEventSynchronize(c.filled[half])); // (its last batch's copies out)- sb = c.stage + (size_t) half * (c.ring_n / P) * c.stage_stride;- std::atomic<bool> bad{false};- c.readers->run(bt.n, [&](int i) {- const int e = l0.host_list[(size_t) (bt.j0 + i)];- if (l0.lay.fmt == gg::kFmtMXFP4 ? !read_mx(l0, e, sb + (size_t) i * c.stage_stride)- : !pk.read_at(l0.region_file[(size_t) e], l0.region_off[(size_t) e],- l0.region_bytes[(size_t) e],- sb + (size_t) i * c.stage_stride))- bad.store(true);- });- if (bad.load()) {- err = "reading layer " + std::to_string(l0.il) + "'s experts from the model's files failed";- return false;- }- }- for (int i = 0; i < bt.n; ++i) {- uint8_t* dst = c.ring + ((size_t) half * (c.ring_n / P) + i) * c.ring_stride;- if (!stream_expert(l0, mi, l0.host_list[(size_t) (bt.j0 + i)], dst, c.cs, err,- sb != nullptr ? sb + (size_t) i * c.stage_stride : nullptr))- return false;- c.stream_bytes += (double) unit(member(l0, mi));- }- CK(cudaEventRecord(c.filled[half], c.cs));- ++S.cp;- S.done.store(S.cp, std::memory_order_relaxed);- }- return true;- }-- // ---- the flow (see Chunk)- // Windows' HIP runtime submits queued work only when the host waits on the GPU: the flow's copies and the compute- // it queued between its waits are submitted at once (cudaStreamQuery does that without waiting); elsewhere nothing- static void kick(cudaStream_t s) {-#if defined(_WIN32) && defined(STRATA_USE_HIP)- (void) cudaStreamQuery(s);- (void) cudaGetLastError();-#else- (void) s;-#endif- }- size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)- return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;- }- size_t flow_stride() const { return flow_region(0) + flow_region(1) + flow_region(2); }- static Chunk::Item& fit(Chunk& c, size_t i) { return c.items[i - c.base]; }- // what member m of l's group reads of matrix r of an expert: gate / up its rows [m n_ff / nm, ..) - one run, from- // this offset in the matrix, this long; the down the whole matrix (its part, the blocks [m n_ff / nm / 32, ..) of- // every row, is DMA'd out of it row by row); alone (nm 1) every matrix whole- size_t mx_read_off(const Layer& l, int m, int r) const {- return r < 2 && l.nm > 1 ? (size_t) m * (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)- : 0;- }- size_t mx_read_len(const Layer& l, int r) const {- return r < 2 && l.nm > 1 ? (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)- : mx_part_bytes(l, r);- }- // the device's member index in L0's group (0 alone)- int flow_mi(const Chunk& c, const Layer& L0) const {- for (int m = 0; m < L0.nm; ++m)- if (member(L0, m).dev == c.dev) return m;- return 0;- }- void flow_push(Chunk& c, const Layer& l, int mi, int k, int e, int v) {- c.items.emplace_back();- Chunk::Item& it = c.items.back();- it.il = l.il;- it.mi = mi;- it.k = k;- it.e = e;- it.v = v;- bool in_ram = !l.ram.empty() && l.ram[(size_t) e] >= 0;- const uint8_t* store_blob = nullptr; // (the RAM store holds it: copied into staging like a RAM-tier slot, no read)- if (!in_ram && store && l.nm == 1) {- const int ss = store->find(l.il, e);- if (ss >= 0) {- store_blob = store->blob(ss);- in_ram = true;- }- }- if (in_ram && ram_pinned && store_blob == nullptr) { // (DMA'd from where it lies)- const uint8_t* p = ram_slots[(size_t) l.ram[(size_t) e]];- for (int r = 0; r < 3; ++r) it.src[r] = p + l.lay.trellis[r] + mx_read_off(l, mi, r);- ++c.it_ram;- return;- }- if (in_ram) // (copied into staging by io, then DMA'd; copy_mode: from its NH parts, flow_pump)- it.ram = store_blob != nullptr ? store_blob- : copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] : ram_slots[(size_t) l.ram[(size_t) e]];- bool hit = false;- if (!in_ram && c.lc != nullptr && l.nm == 1) { // layer-major: from the layer cache, or kept for the chunks after- const size_t i = c.base + c.items.size() - 1, stride = (l.blob + 63) & ~(size_t) 63;- if (c.lc_il != l.il) { // (a layer's first item: the slots the layer before used are its own again- c.lc_il = l.il; // once that layer's items are all read - `after` below)- c.lc_used = 0;- c.lc_n = (int) std::min<size_t>(c.lc_of.size(), c.lc_bytes / stride);- c.lc_first = i;- std::fill(c.lc_of.begin(), c.lc_of.end(), -1);- }- const int j = c.lc_of[(size_t) e];- if (j >= 0) { // (copied into staging by io once its reader is done, then DMA'd)- it.ram = c.lc + (size_t) j * stride;- it.after = c.lc_writer[(size_t) j];- hit = true;- } else if (k + 1 < (int) c.fl_T.size() && c.lc_used < c.lc_n) { // (a later chunk may pick it)- const int jn = c.lc_used++;- c.lc_of[(size_t) e] = jn;- c.lc_writer[(size_t) jn] = (int64_t) i;- it.keep = c.lc + (size_t) jn * stride;- it.after = (int64_t) c.lc_first - 1;- }- }- if (c.direct && l.nm == 1 && !(in_ram && copy_mode)) { // (the ring slot is the destination: filled by flow_pump when its turn comes)- it.direct = true;- it.stage = -1;- it.left.store(3, std::memory_order_relaxed);- ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);- return;- }- it.stage = (int) (c.n_file++ % (uint64_t) c.S);- uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;- for (int r = 0; r < 3; ++r) {- const uint64_t off = l.mx_off[r] + (uint64_t) e * l.mx_stride[r] + mx_read_off(l, mi, r);- it.src[r] = sb + (in_ram || hit ? 0 : off % os::kDirectAlign);- sb += flow_region(r);- }- it.left.store(in_ram && copy_mode && store_blob == nullptr ? 3 * NH : 3, std::memory_order_relaxed);- ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);- }- // the likely experts of (chunk k, layer il) for T tokens, first in its stream: the CPU lane's experts whose chance- // to be picked at least once, 1 - (1 - share)^(picks), is at least DS4_SPEC_Q (0.9: the reads are what takes the- // time, an unpicked one costs a whole read, and the routing is known a few ms after the layer before is done -- // early reads pay where a chunk's long attention keeps the disk otherwise idle, and there nearly every expert is- // that likely), likeliest first, at most what the staging and the ring hold (no more can be read before the layer- // routes); the layer's virtual slots put them first, then its other CPU-lane experts in index order (the order the- // rest of its picks are streamed in)- // (a tensor-parallel group's members each decide their own stream and virtual slots - the same, from the same- // profile and the same routing - and each reads only its part of every expert)- void flow_spec(Chunk& c, int k, int il, int T) {- static const double spec_q = std::getenv("DS4_SPEC_Q") ? std::atof(std::getenv("DS4_SPEC_Q")) : 0.9;- Layer& l = L[(size_t) il];- const int mi = flow_mi(c, l);- int* vs = member(l, mi).vs_pin;- c.spec_k = k;- c.spec_il = il;- c.spec_begin = c.base + c.items.size();- const uint32_t* pr = prof.data() + (size_t) il * g.n_exp;- double sum = 0;- for (int e = 0; e < l.n_exp; ++e) sum += pr[e];- std::vector<std::pair<double, int>> cand;- const double picks = (double) l.k_exp * T;- // (early reads only while they hit: once a run's likely ones went unpicked more often than DS4_SPEC_WASTE- // (0.2) of the time, the rest of the run reads only what the routing picked - an unpicked one costs a whole- // read, and how well the profile guesses depends on the PC's history and the prompt: V100, 64 tokens - 376- // of 921 likely ones unpicked, 7 GB of 84 read for nothing; Radeon 8065S, 128 and 512 tokens - 44 of 945 and- // 21 of 1568, the early reads 9-12% faster there)- static const double waste = std::getenv("DS4_SPEC_WASTE") ? std::atof(std::getenv("DS4_SPEC_WASTE")) : 0.2;- const bool early = c.it_spec < 2u * (uint64_t) c.ring_n || (double) c.it_spec_unused <= waste * (double) c.it_spec;- for (int e : l.host_list) {- if (!early) break;- const double p = sum > 0 ? pr[e] / sum : 0.0;- const double q = p >= 1.0 ? 1.0 : 1.0 - std::exp(picks * std::log1p(-p));- if (q >= spec_q && spec_q <= 1.0) cand.push_back({q, e});- }- std::stable_sort(cand.begin(), cand.end(), [](const std::pair<double, int>& a, const std::pair<double, int>& b) {- return a.first > b.first;- });- if (cand.size() > (size_t) (c.S + c.ring_n)) cand.resize((size_t) (c.S + c.ring_n));- std::vector<char> is((size_t) l.n_exp, 0);- int j = 0;- for (const auto& x : cand) {- vs[x.second] = l.n_slots + j++;- is[(size_t) x.second] = 1;- }- for (int e : l.host_list)- if (!is[(size_t) e]) vs[e] = l.n_slots + j++;- for (const auto& x : cand) flow_push(c, l, mi, k, x.second, vs[x.second]);- c.n_spec = (int) cand.size();- c.it_spec += cand.size();- }- // (chunk k, layer il) routed (bounds b by virtual slot): the rest of its picks after its likely ones- void flow_rest(Chunk& c, int k, int il, const std::vector<int>& b) {- const Layer& l = L[(size_t) il];- const int mi = flow_mi(c, l);- const int* vs = member(l, mi).vs_pin;- for (size_t i = c.spec_begin; i < c.spec_begin + (size_t) c.n_spec; ++i) {- const int v = fit(c, i).v;- if (b[(size_t) v + 1] == b[(size_t) v]) ++c.it_spec_unused;- }- for (int e : l.host_list) { // (index order: their virtual slots ascending)- const int v = vs[e];- if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);- }- }- // the next (chunk, layer) of the device after (k, il) in the run's order (false: none)- bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {- const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);- const size_t i = (size_t) (it - c.fl_layers.begin());- if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)- if (k + 1 < (int) c.fl_T.size()) {- k2 = k + 1;- il2 = il;- return true;- }- if (i + 1 >= c.fl_layers.size()) return false;- k2 = 0;- il2 = c.fl_layers[i + 1];- return true;- }- if (i + 1 < c.fl_layers.size()) {- k2 = k;- il2 = c.fl_layers[i + 1];- return true;- }- if (k + 1 < (int) c.fl_T.size() && !c.fl_layers.empty()) {- k2 = k + 1;- il2 = c.fl_layers[0];- return true;- }- return false;- }- // the reads (in order, each into its staging slot once that slot's last DMA is done) and the DMAs (in order, each- // into its ring slot once that slot's last use is queued: the copy stream waits for it) as far as they can go now- bool flow_pump(Chunk& c, std::string& err) {- const size_t end = c.base + c.items.size();- IoQueue::Job jobs[6];- while (c.n_read < end) {- Chunk::Item& it = fit(c, c.n_read);- if (it.after >= 0) { // layer-major: a slot's expert copied out of it once its reader is done; a slot- // written again once the layer before's items are all read (none still copies out of it)- bool wait = false;- if (it.keep == nullptr)- wait = (size_t) it.after >= c.base && fit(c, (size_t) it.after).left.load(std::memory_order_acquire) != 0;- else {- while (c.lc_chk < c.n_read &&- (c.lc_chk < c.base || fit(c, c.lc_chk).left.load(std::memory_order_acquire) == 0))- ++c.lc_chk;- wait = c.lc_chk <= (size_t) it.after;- }- if (wait) break;- }- if (it.direct) {- const size_t R = (size_t) c.ring_n, r = c.n_read % R;- if (c.n_read >= R) { // the slot's last use has to be computed - and finished: the CPU writes it- if (c.n_comp <= c.n_read - R) break;- CK(cudaEventSynchronize(c.ring_free[r]));- }- const Layer& l = L[(size_t) it.il];-#if defined(STRATA_METAL)- uint8_t* dst = (uint8_t*) strata::gpu::host_view(c.ring + r * c.ring_stride); // (device addresses are the GPU's: the CPU writes through the host view)-#else- uint8_t* dst = c.ring + r * c.ring_stride;-#endif- const x3::ExpertLayout& lay = l.lay;- int nj = 0;- for (int q = 0; q < 3; ++q) {- IoQueue::Job& j = jobs[nj++];- j = IoQueue::Job{};- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[q]];- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[q]];- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[q]];- j.off = l.mx_off[q] + (uint64_t) it.e * l.mx_stride[q];- j.len = mx_part_bytes(l, q);- j.dst = dst + lay.trellis[q];- j.exact = true;- j.left = &it.left;- j.bad = &it.bad;- j.mem = it.ram != nullptr ? it.ram + lay.trellis[q] : nullptr;- j.map = pk.at(l.mx_file[q], j.off);- j.keep = it.keep != nullptr ? it.keep + lay.trellis[q] : nullptr;- }- c.io->push(jobs, nj);- } else if (it.stage >= 0) {- const int64_t prev = c.stage_item[(size_t) it.stage];- if (prev >= 0) {- if ((size_t) prev >= c.n_dma) break;- const cudaError_t q = cudaEventQuery(c.stage_ev[(size_t) it.stage]);- if (q == cudaErrorNotReady) break;- CK(q);- }- c.stage_item[(size_t) it.stage] = (int64_t) c.n_read;- const Layer& l = L[(size_t) it.il];- uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;- const int cs = copy_mode && it.ram != nullptr ? l.ram[(size_t) it.e] : -1; // (a copy's slot)- int nj = 0;- for (int r = 0; r < 3; ++r) {- if (cs >= 0) { // a copy: each matrix's NH parts of rows, side by side into staging- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);- for (int X = 0; X < NH; ++X) {- IoQueue::Job& j = jobs[nj++];- j = IoQueue::Job{};- j.len = nb;- j.dst = sb + (size_t) X * nb;- j.left = &it.left;- j.bad = &it.bad;- j.mem = ram_part[(size_t) cs][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];- }- sb += flow_region(r);- continue;- }- IoQueue::Job& j = jobs[nj++];- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];- j.off = l.mx_off[r] + (uint64_t) it.e * l.mx_stride[r] + mx_read_off(l, it.mi, r);- j.len = mx_read_len(l, r);- j.dst = sb;- j.left = &it.left;- j.bad = &it.bad;- j.mem = it.ram != nullptr ? it.ram + l.lay.trellis[r] + mx_read_off(l, it.mi, r) : nullptr;- j.map = pk.at(l.mx_file[r], j.off);- j.keep = it.keep != nullptr ? it.keep + l.lay.trellis[r] : nullptr;- sb += flow_region(r);- }- c.io->push(jobs, nj);- }- ++c.n_read;- }- const size_t R = (size_t) c.ring_n, dma0 = c.n_dma;- struct Kick { // (every way out: what was queued is submitted)- cudaStream_t s;- const size_t &a, b;- ~Kick() {- if (a != b) kick(s);- }- } kq{c.cs, c.n_dma, dma0};- while (c.n_dma < c.n_read && c.n_dma < c.n_comp + R) {- Chunk::Item& it = fit(c, c.n_dma);- if (it.left.load(std::memory_order_acquire) != 0) break;- if (it.bad.load(std::memory_order_acquire)) {- err = "reading layer " + std::to_string(it.il) + " expert " + std::to_string(it.e) +- " from the model's files failed";- return false;- }- const size_t r = c.n_dma % R;- if (c.n_dma >= R) CK(cudaStreamWaitEvent(c.cs, c.ring_free[r], 0));- const Layer& l = L[(size_t) it.il];- uint8_t* dst = c.ring + r * c.ring_stride;- const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;- if (it.direct) { // (already in the slot)- CK(cudaEventRecord(c.ring_fill[r], c.cs));- c.stream_bytes += (double) unit(l);- ++c.n_dma;- continue;- }- for (int p = 0; p < 2; ++p)- CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));- if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)- const int t2 = gg::expert_type(l.lay, 2);- const size_t dr = gg::type_row_bytes(t2, g.ff), dh = gg::type_row_bytes(t2, g.ff / l.nm);- CK(cudaMemcpy2DAsync(dst + lay.trellis[2], dh, it.src[2] + (size_t) it.mi * dh, dr, dh, (size_t) g.dim,- cudaMemcpyHostToDevice, c.cs));- } else- CK(cudaMemcpyAsync(dst + lay.trellis[2], it.src[2], mx_part_bytes(l, 2), cudaMemcpyHostToDevice, c.cs));- CK(cudaEventRecord(c.ring_fill[r], c.cs));- if (it.stage >= 0) CK(cudaEventRecord(c.stage_ev[(size_t) it.stage], c.cs));- c.stream_bytes += (double) unit(l);- ++c.n_dma;- }- return true;- }- // layer L0 of chunk k routed (bounds b, the device's copy d_bounds): its streamed experts as they land, the VRAM- // ones in shares between them (the device works while the reads run)- bool flow_layer(Dev& d, Chunk& c, Layer& L0, int mi, int k, int T, const std::vector<int>& b, std::string& err) {- cudaStream_t s = d.st;- const uint8_t* vbase = member(L0, mi).vram_base; // (a member's: its part of each VRAM expert, L0's slots)- if (c.spec_k != k || c.spec_il != L0.il) {- err = "the prompt path's expert stream is out of step (layer " + std::to_string(L0.il) + ")";- return false;- }- flow_rest(c, k, L0.il, b);- const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;- int k2 = 0, il2 = 0;- if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, c.fl_T[(size_t) k2]);- const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);- int v_done = 0;- while (c.n_comp < le) {- // a batch's bounds are fixed by the stream alone (bmax items, never across the ring's end), never by what- // has landed: the experts' outputs are summed batch after batch (the VRAM ones' shares between them), so- // the sums come out the same on every run whatever the reads' timing- const size_t i0 = c.n_comp, i1 = std::min({le, i0 + bmax, (i0 / R + 1) * R});- if (!flow_pump(c, err)) return false;- if (c.n_dma < i1) { // not all of the batch landed: wait for a read, or for a staging slot's copy- const double tw = now_ms();- kick(s);- if (c.n_read > c.n_dma) c.io->wait_any(500);- else {- const Chunk::Item& it = fit(c, c.n_read);- if (it.stage >= 0) CK(cudaEventSynchronize(c.stage_ev[(size_t) it.stage]));- }- c.wait_ms += now_ms() - tw;- continue;- }- const int n = (int) (i1 - i0);- int* bc = c.bc_pin + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;- int* bcd = c.bc_dev + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;- ++c.n_batch;- // (the items' virtual slots ascend and the slots between them have no rows: their bounds side by side)- for (int m = 0; m < n; ++m) bc[m] = b[(size_t) fit(c, i0 + (size_t) m).v];- bc[n] = b[(size_t) fit(c, i1 - 1).v + 1];- if (bc[n] > bc[0]) {- CK(cudaStreamWaitEvent(s, c.ring_fill[(i1 - 1) % R], 0));- CK(cudaMemcpyAsync(bcd, bc, (size_t) (n + 1) * 4, cudaMemcpyHostToDevice, s));- if (!run_set(d, c, L0, c.ring + (i0 % R) * c.ring_stride, 0, c.ring_stride, bcd, bc, 0, n, T, err))- return false;- }- for (size_t i = i0; i < i1; ++i) CK(cudaEventRecord(c.ring_free[i % R], s));- kick(s);- c.n_comp = i1;- if (!flow_pump(c, err)) return false;- const int v_to = (int) ((int64_t) L0.n_slots * (int64_t) (c.n_comp - lb) / (int64_t) std::max<size_t>(1, n_lay));- if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, v_to, T, err)) return false;- v_done = v_to;- }- if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;- while (c.base < c.n_comp && c.items.size() > 4096) { // (done with: read, copied and used)- c.items.pop_front();- ++c.base;- }- return true;- }-- // the experts [i0, i1) of a sorted set (blobs at wbase + i * stride; bounds b on the host, d_bounds on the device)- // over the chunk's rows, in pieces of at most C rows, each added into acc in pick order- bool run_set(Dev& d, Chunk& c, const Layer& l, const uint8_t* wbase, int wb0, size_t stride, const int* d_bounds,- const int* b, int i0, int i1, int T, std::string& err) {- const int D = g.dim, FF = l.ffn;- while (i0 < i1) {- int j = i0 + 1;- while (j < i1 && b[j + 1] - b[i0] <= c.C) ++j;- const int a = b[i0], nrows = b[j] - a;- int max_rows = 0;- for (int i = i0; i < j; ++i) max_rows = std::max(max_rows, b[i + 1] - b[i]);- if (nrows > 0) {- x3::SetArgs sa;- sa.wbase = wbase + (size_t) (i0 - wb0) * stride; // (wbase: set member wb0's blob)- sa.stride = stride;- sa.lay = l.nm > 1 ? l.hlay : l.lay;- sa.n_exp = j - i0;- sa.bounds = d_bounds + i0;- sa.row0 = a;- sa.max_rows = max_rows;- sa.hbounds = b + i0;- sa.total_rows = nrows;- sa.n_embd = D;- sa.n_ff = FF;- sa.limit = g.lim;- if (sa.lay.fmt == gg::kFmtMXFP4) {- gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);- gg::set_down(sa, c.h16, FF, c.y, D, d.st);- } else {- x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);- x3::set_down(sa, c.h16, FF, c.y, D, d.st);- }- if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {- std::vector<int> bb(b + i0, b + j + 1);- std::vector<uint16_t> hv((size_t) nrows * FF);- std::vector<float> yv((size_t) nrows * D);- CK(cudaStreamSynchronize(d.st));- CK(cudaMemcpy(hv.data(), c.h16, hv.size() * 2, cudaMemcpyDeviceToHost));- CK(cudaMemcpy(yv.data(), c.y, yv.size() * 4, cudaMemcpyDeviceToHost));- std::lock_guard<std::mutex> lk(dump_mu);- const int64_t hdr[6] = {l.il, a, nrows, j - i0, wbase == c.ring || (wbase > c.ring &&- wbase < c.ring + (size_t) c.ring_n * c.ring_stride) ? 1 : 0, max_rows};- std::fwrite(hdr, 8, 6, dump_y);- std::fwrite(bb.data(), 4, bb.size(), dump_y);- std::fwrite(hv.data(), 2, hv.size(), dump_y);- std::fwrite(yv.data(), 4, yv.size(), dump_y);- std::fflush(dump_y);- }- moe_combine(c.acc, c.y, c.pos, c.rw, T, l.k_exp, a, a + nrows, D, d.st);- }- i0 = j;- }- (void) err;- return true;- }-- bool load_dense(int di, std::string& err) {- Dev& d = devs[(size_t) di];- cudaSetDevice(d.id);- size_t* acct = &d.weights;- for (Layer& l0 : L)- for (int mi = 0; mi < l0.nm; ++mi) {- Layer& l = member(l0, mi);- if (l.dev != di) continue;- // (a group's member: its share of the heads (wq_b columns, the sinks), of wo_a's groups and wo_b's rows,- // of the shared expert's FFN; the rest whole)- const int nm = l0.nm, na = g.heads / std::max(1, l.ahn), ha = l.ah0 / std::max(1, l.ahn);- const std::string& P = l.name;- const std::string A = P + "attn.";- bool ok = mat(d, A + "wq_a", l.wq_a, err, acct) &&- (l.ahn == 0 || mat(d, A + "wq_b", l.wq_b, err, acct, ha, na, true)) &&- mat(d, A + "wkv", l.wkv, err, acct) &&- (l.ahn == 0 || mat(d, A + "wo_b", l.wo_b, err, acct, ha, na, false)) &&- f32(d, A + "q_norm.weight", l.q_norm, err, acct, (size_t) g.q_lora) &&- f32(d, A + "kv_norm.weight", l.kv_norm, err, acct, (size_t) g.hd) &&- f32(d, A + "attn_sink", l.sink, err, acct, (size_t) g.heads) &&- f32(d, P + "attn_norm.weight", l.attn_norm, err, acct, (size_t) g.dim) &&- f32(d, P + "ffn_norm.weight", l.ffn_norm, err, acct, (size_t) g.dim) &&- f32(d, P + "hc_attn_fn", l.hca_fn, err, acct) && f32(d, P + "hc_attn_scale", l.hca_scale, err, acct) &&- f32(d, P + "hc_attn_base", l.hca_base, err, acct) && f32(d, P + "hc_ffn_fn", l.hcf_fn, err, acct) &&- f32(d, P + "hc_ffn_scale", l.hcf_scale, err, acct) &&- f32(d, P + "hc_ffn_base", l.hcf_base, err, acct) &&- f32(d, P + "ffn.gate.weight", l.gate_w, err, acct, (size_t) l.n_exp * g.dim) &&- f32(d, P + "ffn.gate.bias", l.gate_bias, err, acct, (size_t) l.n_exp) &&- (l.mtp || img_tok < 0 || pk.get(P + "ffn.gate.bias_vl") == nullptr ||- f32(d, P + "ffn.gate.bias_vl", l.gate_bias_vl, err, acct, (size_t) l.n_exp)) &&- mat(d, P + "ffn.shared_experts.w1", l.s1, err, acct, mi, nm, true, l.il % nm) &&- mat(d, P + "ffn.shared_experts.w2", l.s2, err, acct, mi, nm, false, l.il % nm) &&- mat(d, P + "ffn.shared_experts.w3", l.s3, err, acct, mi, nm, true, l.il % nm);- if (!ok) return false;- l.sink += (size_t) l.ah0;- for (int gi = 0; gi < l.ogn; ++gi)- if (!mat(d, A + "wo_a.slice." + std::to_string(l.og0 + gi), l.wo_a[gi], err, acct)) return false;- if (l.kv_src) {- const std::string C = A + "compressor.";- if (!mat(d, C + "wkv", l.c_wkv, err, acct) || !f32(d, C + "norm.weight", l.c_norm, err, acct)) return false;- if (l.ratio > 1 && !mat(d, C + "wgate", l.c_wgate, err, acct)) return false;- if (l.ratio > 1 && (!dalloc(d, l.c_kv, (size_t) l.ratio * g.hd, err) ||- !dalloc(d, l.c_score, (size_t) l.ratio * g.hd, err)))- return false;- }- if (l.ix_src) {- const std::string I = A + "indexer.";- if (!mat(d, I + "wq_b", l.i_wq_b, err, acct) || !f16raw(d, I + "weights_proj.weight", l.i_wproj, err, acct))- return false;- if (l.owns_k && (!mat(d, I + "wk", l.i_wk, err, acct) || !f32(d, I + "k_norm.weight", l.ik_norm, err, acct)))- return false;- }- if (l.eg >= 0) {- if (!mat(d, P + "engram.wkv", l.eg_wkv, err, acct)) return false;- if (pk.get(P + "engram.gate.weight") != nullptr) { // a GGUF: q_weight * k_weight, folded already- if (!f32(d, P + "engram.gate.weight", l.eg_w, err, acct, (size_t) g.hc * g.dim)) return false;- } else {- float *qw = nullptr, *kw = nullptr;- if (!f32(d, P + "engram.q_weight", qw, err, acct) || !f32(d, P + "engram.k_weight", kw, err, acct))- return false;- std::vector<float> a((size_t) g.hc * g.dim), b(a.size());- CK(cudaMemcpy(a.data(), qw, a.size() * 4, cudaMemcpyDeviceToHost));- CK(cudaMemcpy(b.data(), kw, b.size() * 4, cudaMemcpyDeviceToHost));- for (size_t i = 0; i < a.size(); ++i) a[i] *= b[i]; // only ever used as a product- CK(cudaMemcpy(qw, a.data(), a.size() * 4, cudaMemcpyHostToDevice));- l.eg_w = qw;- }- }- if (!dalloc(d, l.win, (size_t) g.win * win_rb(), err, &d.caches)) return false;- if (g.n_mtp > 0 && !l.mtp) {- if (!dalloc(d, l.ring_bak, (size_t) kWin * win_rb(), err)) return false;- if (l.kv_src && l.ratio > 1 && (!dalloc(d, l.c_snap, (size_t) 2 * l.ratio * g.hd, err) ||- !dalloc(d, l.c_rows, (size_t) 2 * kWin * g.hd, err)))- return false;- }- if (!dalloc(d, l.vram_tab, (size_t) l.n_exp, err, acct)) return false;- }- if (d.grp == devs[(size_t) head_dev].grp) { // the head: each member of its group a share of the columns- const int hn = head_nm();- if (!mat(d, "head", d.head, err, acct, d.mi, hn, true) || !f32(d, "norm.weight", d.norm_w, err, acct, (size_t) g.dim))- return false;- if (g.n_mtp > 0 && !f32(d, "mtp." + std::to_string(g.n_mtp - 1) + ".norm.weight", d.d_norm, err, acct, (size_t) g.dim))- return false;- if (!dalloc(d, d.logits_part, (size_t) kWin * ((g.vocab / 128 + hn - 1) / hn * 128), err)) return false;- if (g.n_mtp > 0 && hn > 1) { // the markov steps over the group (mk_split): this member's rows of the head- const Tensor* th = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.head.weight");- const int R = g.d_rank;- head_cols(hn, d.mi, d.mk_v0, d.mk_vn); // (its share of the head's columns: the same rows)- // F16 as the pack holds it, or a GGUF drafter's BF16 rounded to F16 as f16raw rounds d_mk_head (the same- // values as the lead's whole rows); another type: no mk_split (the lead's whole rows)- if (th != nullptr && (th->dtype == "F16" || th->dtype == "BF16") && th->bytes == (uint64_t) g.vocab * R * 2) {- std::vector<uint16_t> rows((size_t) d.mk_vn * R);- if (!pk.read_at(th->file, th->off + (uint64_t) d.mk_v0 * R * 2, rows.size() * 2, rows.data())) {- err = "the markov head: read failed";- return false;- }- if (th->dtype == "BF16")- for (uint16_t& h : rows) h = strata::kernels::f16_from_f32(bf16_to_f32(h));- if (!dalloc(d, d.mk_share, rows.size(), err, acct) || !dalloc(d, d.mk_erow, (size_t) R, err) ||- !dalloc(d, d.mk_ints, (size_t) kWin + 2, err))- return false;- CK(cudaMemcpy(d.mk_share, rows.data(), rows.size() * 2, cudaMemcpyHostToDevice));- }- }- }- if (di == head_dev) {- head = d.head;- norm_w = d.norm_w;- d_norm = d.d_norm;- if (g.n_mtp > 0) {- const std::string Z = "mtp." + std::to_string(g.n_mtp - 1) + ".";- if (!f32(d, Z + "confidence_head.proj.weight", d_conf, err, acct, (size_t) (g.dim + g.d_rank)))- return false;- const Tensor* te = pk.get(Z + "markov_head.embed.weight");- const Tensor* th = pk.get(Z + "markov_head.head.weight");- if (te == nullptr || th == nullptr || te->dtype != "BF16" || (th->dtype != "F16" && th->dtype != "BF16")) {- err = Z + "markov_head: not the BF16 embed and F16 / BF16 head expected";- return false;- }- if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||- !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))- return false;- }- }- if (g.n_mtp > 0 && di == d_dev) {- if (!mat(d, "mtp.0.main_proj", d_main_proj, err, acct) ||- !f32(d, "mtp.0.main_norm.weight", d_main_norm, err, acct, (size_t) g.dim))- return false;- }- return true;- }--#if defined(STRATA_METAL)- // The most the VRAM experts may add on a Mac: the engine's anonymous memory (Metal buffers + host memory, all counted) stays- // under the cap metal/src/engine/arena_budget.h derived from the freezes of the 128 GB M5 Max (macOS keeps a file-cache floor and- // compresses / swaps anonymous memory past it): max(15% of the RAM, 40% of the RAM, the floor + 12 GB) left to macOS, at most- // the GPU working set; what the process holds now (task_info phys_footprint) is already inside it.- double metal_expert_cap() const {- uint64_t phys = 0;- size_t len = sizeof phys;- sysctlbyname("hw.memsize", &phys, &len, nullptr, 0);- unsigned long long pages = 0;- size_t pl = sizeof pages;- double floor_b = 0;- if (sysctlbyname("vm.vm_page_filecache_min", &pages, &pl, nullptr, 0) == 0) floor_b = (double) pages * (double) vm_page_size;- size_t fr = 0, tot = 0;- cudaMemGetInfo(&fr, &tot);- const double gb = 1e9, P = (double) phys;- double keep = std::max(7.0 * gb, 0.15 * P);- if (const char* e = std::getenv("STRATA_METAL_KEEP_FREE_GB")) keep = std::atof(e) * gb;- double share = 0.40;- if (const char* e = std::getenv("STRATA_METAL_LEAVE_SHARE")) share = std::atof(e);- double leave = keep;- if (P >= 64.0 * gb) leave = std::max({leave, share * P, floor_b > 0 ? floor_b + 12.0 * gb : 0.0});- const double cap = std::min((double) tot, P - leave);- task_vm_info_data_t ti;- mach_msg_type_number_t cnt = TASK_VM_INFO_COUNT;- double foot = 0;- if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &ti, &cnt) == KERN_SUCCESS) foot = (double) ti.phys_footprint;- return std::max(4.0 * gb, cap - foot - 1.0 * gb);- }-#endif-#if defined(STRATA_METAL)- // Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it- // like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).- bool zc_try(Layer& l, int e) {- static const bool on = std::getenv("DS4_ZC") != nullptr; // (measured: no faster than the CPU lane - cold pages come from the SSD either way)- if (!on || l.lay.fmt != gg::kFmtMXFP4 || l.nm != 1 || l.vm.empty()) return false;- if (l.vm[(size_t) e][0].raw != nullptr) return true;- uint64_t a[3];- for (int r = 0; r < 3; ++r) {- a[r] = ::strata::dsv41::metal_host::zc_map(mx_mat(l, e, r), mx_part_bytes(l, r));- if (a[r] == 0) return false;- }- for (int r = 0; r < 3; ++r) {- x3::Mat& m = l.vm[(size_t) e][(size_t) r];- m = x3::Mat{};- m.K = gg::expert_type(l.lay, r);- m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;- m.raw = (const void*) a[r];- m.k = r < 2 ? g.dim : g.ff;- m.n = r < 2 ? g.ff : g.dim;- }- return true;- }-#endif- void set_vm(Layer& l, int e, int slot) {- uint8_t* b = l.vram_base + (size_t) slot * l.blob;- if (l.lay.fmt == gg::kFmtMXFP4) {- for (int r = 0; r < 3; ++r) {- x3::Mat& m = l.vm[(size_t) e][(size_t) r];- m = x3::Mat{};- m.K = gg::expert_type(l.lay, r); // (a low-bit type: its GEMV, ds4_iq's)- m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;- m.raw = b + l.lay.trellis[r];- m.k = r < 2 ? g.dim : g.ff;- m.n = r < 2 ? g.ff : g.dim;- }- return;- }- for (int r = 0; r < 3; ++r) {- x3::Mat& m = l.vm[(size_t) e][(size_t) r];- m.trellis = (const uint32_t*) (b + l.lay.trellis[r]);- m.suh = (const uint16_t*) (b + l.lay.suh[r]);- m.svh = (const uint16_t*) (b + l.lay.svh[r]);- m.k = r < 2 ? g.dim : g.ff;- m.n = r < 2 ? g.ff : g.dim;- m.K = l.lay.K[r];- m.cb = 2;- m.ld = 0;- }- }- // an expert's VRAM blob -> its CPU-lane part X, and its NH parts -> the blob (the cache's swaps)- void blob_to_part(const Layer& l, const uint8_t* blob, int X, uint8_t* dst) const {- for (int r = 0; r < 3; ++r) {- const size_t tile = (size_t) 32 * l.lay.K[r];- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;- const uint8_t* tr = blob + l.lay.trellis[r];- for (int sl = 0; sl < k / 16; ++sl)- std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,- (size_t) nth * tile);- std::memcpy(dst + l.hl.suh[r], blob + l.lay.suh[r], (size_t) k * 2);- std::memcpy(dst + l.hl.svh[r], blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);- }- }- void parts_to_blob(const Layer& l, const std::array<uint8_t*, 4>& parts, uint8_t* blob) const {- for (int r = 0; r < 3; ++r) {- const size_t tile = (size_t) 32 * l.lay.K[r];- const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;- uint8_t* tr = blob + l.lay.trellis[r];- for (int X = 0; X < NH; ++X)- for (int sl = 0; sl < k / 16; ++sl)- std::memcpy(tr + ((size_t) sl * nt + (size_t) X * nth) * tile,- parts[(size_t) X] + l.hl.trellis[r] + (size_t) sl * nth * tile, (size_t) nth * tile);- std::memcpy(blob + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);- for (int X = 0; X < NH; ++X)- std::memcpy(blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, parts[(size_t) X] + l.hl.svh[r],- (size_t) l.hl.n[r] * 2);- }- }-- // an expert's blob -> member m's part (gate / up columns [ff0, ff0 + ffn), down rows of the same), and back- void blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {- if (l.lay.fmt == gg::kFmtMXFP4) return mx_blob_to_half(l, blob, m, dst);- const Layer& x = member(l, m);- const int f0 = x.ff0, fn = x.ffn;- for (int r = 0; r < 2; ++r) {- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;- for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)- std::memcpy(dst + x.hlay.trellis[r] + sl * nth * tile,- blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile, nth * tile);- std::memcpy(dst + x.hlay.suh[r], blob + l.lay.suh[r], (size_t) g.dim * 2);- std::memcpy(dst + x.hlay.svh[r], blob + l.lay.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);- }- const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];- std::memcpy(dst + x.hlay.trellis[2], blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, (size_t) (fn / 16) * row);- std::memcpy(dst + x.hlay.suh[2], blob + l.lay.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);- std::memcpy(dst + x.hlay.svh[2], blob + l.lay.svh[2], (size_t) g.dim * 2);- }- void half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {- if (l.lay.fmt == gg::kFmtMXFP4) return mx_half_to_blob(l, src, m, blob);- const Layer& x = member(l, m);- const int f0 = x.ff0, fn = x.ffn;- for (int r = 0; r < 2; ++r) {- const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;- for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)- std::memcpy(blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile,- src + x.hlay.trellis[r] + sl * nth * tile, nth * tile);- if (m == 0) std::memcpy(blob + l.lay.suh[r], src + x.hlay.suh[r], (size_t) g.dim * 2);- std::memcpy(blob + l.lay.svh[r] + (size_t) f0 * 2, src + x.hlay.svh[r], (size_t) fn * 2);- }- const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];- std::memcpy(blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, src + x.hlay.trellis[2], (size_t) (fn / 16) * row);- std::memcpy(blob + l.lay.suh[2] + (size_t) f0 * 2, src + x.hlay.suh[2], (size_t) fn * 2);- if (m == 0) std::memcpy(blob + l.lay.svh[2], src + x.hlay.svh[2], (size_t) g.dim * 2);- }-- bool place_experts(std::string& err) {- // (Windows, an integrated GPU: what its experts may commit - commit_room - and what that kept off the GPU)- double room = 1e300, kept = 0;- for (const Dev& d : devs)- if (commit_charged(d)) room = commit_room();- const double room0 = room;- // per group: what its tightest member has left (each member holds its part of every VRAM expert of the- // group's layers, its unit bytes a slot - counted at the members' mean: an uneven split's odd blocks go round- // the members layer by layer, so each member's total comes to about the mean's)- auto ucount = [](const Layer& l) {- double s = 0;- for (int m = 0; m < l.nm; ++m) s += (double) unit(member(l, m));- return s / l.nm;- };- for (size_t gi = 0; gi < groups.size(); ++gi) {- double budget = 1e300;- bool charged = false;- for (int m : groups[gi].mem) {- cudaSetDevice(devs[(size_t) m].id);- size_t fr = 0, tot = 0;- CK(cudaMemGetInfo(&fr, &tot));- budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20))) *- o.vram_experts);-#if defined(STRATA_METAL)- if (o.vram_experts_auto && std::getenv("DS4_NO_AUTO_VX") == nullptr) {- const double cap = metal_expert_cap();- if (cap < budget) {- std::fprintf(stderr, "ds4: expert share: %.1f GB of VRAM experts (the RAM cap leaves %.1f GB; --vram-experts sets it)\n",- cap / 1e9, cap / 1e9);- budget = cap;- }- }-#endif- charged = charged || commit_charged(devs[(size_t) m]);- }- if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)- double all = 0;- for (const Layer& l : L)- if (l.grp == (int) gi) all += (double) unit(l) * l.n_exp;- const double nm = (double) groups[gi].mem.size(), capped = std::min(budget, room / nm);- kept += std::max(0.0, std::min(budget, all) - std::min(capped, all)) * nm;- budget = capped;- room = std::max(0.0, room - std::min(budget, all) * nm);- }- double budget_t = budget; // the drafter's experts first, all of them (every draft uses its blocks)- for (Layer& l : L)- if (l.grp == (int) gi && l.mtp) {- l.n_vram = std::min(l.n_exp, (int) std::floor(budget_t / (double) ucount(l)));- budget_t -= (double) l.n_vram * ucount(l);- }- double want = 0;- for (const Layer& l : L)- if (l.grp == (int) gi && !l.mtp && !l.cpu_only) want += (double) ucount(l) * l.n_exp;- const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;- for (Layer& l : L) {- if (l.grp != (int) gi || l.mtp) continue;- if (l.cpu_only) { // (its type has no device kernel: the CPU lane reads every expert of it)- l.n_vram = 0;- continue;- }- l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));- // never more than fits: the floor of each layer's share can still round up the total- while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;- }- double used = 0;- for (Layer& l : L)- if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;- // what the floors left: whole experts, layer by layer- for (Layer& l : L) {- if (l.grp != (int) gi || l.cpu_only) continue;- while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {- ++l.n_vram;- used += (double) ucount(l);- }- }- }- if (kept > 0) { // the commit kept experts off the GPU: say so, and the page file that keeps them on it- const os::Commit c = os::commit();- const double gib = (double) (1ull << 30);- const double pf = c.limit != ~uint64_t{0} && c.ram != ~uint64_t{0} && c.limit > c.ram ? (double) (c.limit - c.ram) : 0.0;- commit_kept = kept;- pagefile_hint = std::ceil((pf + kept) / gib);- std::fprintf(stderr,- "ds4: WARNING: %.1f GB of experts stay off the GPU - Windows charges the GPU's memory to its "- "commit (RAM + page file) and can commit only %.1f GB more for them (%.0f GB kept for the "- "other programs). A page file of %.0f GB (%.0f MB; now %.0f GB) keeps them on the GPU: System "- "> About > Advanced system settings > Performance > Advanced > Virtual memory, then restart\n",- kept / 1e9, room0 / 1e9, commit_margin() / 1e9, pagefile_hint, pagefile_hint * 1024.0, pf / gib);- } else if (room0 < 1e300 && o.verbose) {- std::fprintf(stderr, "ds4: Windows' commit has room for the GPU's experts (%.1f GB beside %.0f GB for the other "- "programs)\n", room0 / 1e9, commit_margin() / 1e9);- }- for (Layer& l : L) {- // the hottest experts of the profile first, then by index- std::vector<int> order((size_t) l.n_exp);- std::iota(order.begin(), order.end(), 0);- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;- std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return p[a] > p[b]; });- l.slot.assign((size_t) l.n_exp, -1);- l.host.assign((size_t) l.n_exp, nullptr);- // a layer with experts on both sides keeps one VRAM slot free for the cache's swaps- l.n_slots = std::min(l.n_vram, l.n_exp);- if (o.expert_cache && l.n_slots >= 2 && l.n_slots < l.n_exp) { // (every expert fits: nothing to swap)- l.n_vram = l.n_slots - 1;- l.spare_vram = l.n_slots - 1;- } else {- l.n_vram = l.n_slots;- l.spare_vram = -1;- }- for (int i = 0; i < l.n_vram; ++i) l.slot[(size_t) order[(size_t) i]] = i;- for (int m = 0; m < l.nm && l.n_slots > 0; ++m) {- Layer& x = member(l, m);- Dev& d = devs[(size_t) x.dev];- cudaSetDevice(d.id);- void* v = nullptr;- if (cudaMalloc(&v, (size_t) l.n_slots * unit(x)) != cudaSuccess) {- err = "cudaMalloc of layer " + std::to_string(l.il) + "'s " + std::to_string(l.n_slots) +- " experts on CUDA" + std::to_string(d.id) + " failed";- return false;- }- d.allocs.push_back(v);- d.experts += (size_t) l.n_slots * unit(x);- x.vram_base = (uint8_t*) v;- }- l.heat.assign((size_t) l.n_exp, 0.0f);- { // the profile as the starting heat: its counts scaled to ~512 hits a layer- double sum = 0;- for (int e = 0; e < l.n_exp; ++e) sum += p[e];- const double sc = sum > 0 ? 512.0 / sum : 0.0;- for (int e = 0; e < l.n_exp; ++e) l.heat[(size_t) e] = (float) (p[e] * sc);- }- const int n_host = l.n_exp - l.n_vram;- { // the CPU lane's parts of an expert (in place: the whole expert, its pieces where the file has them)- size_t off = 0;- for (int r = 0; r < 3; ++r) {- const int k = r < 2 ? g.dim : g.ff, nf = r < 2 ? g.ff : g.dim, n = nf / NH;- l.hl.k[r] = k;- l.hl.n[r] = n;- l.hl.trellis[r] = file_experts ? l.piece_src[r][0] : off;- off += align256((size_t) (k / 16) * (n / 16) * 32 * l.lay.K[r]);- l.hl.suh[r] = file_experts ? l.piece_src[r][1] : off;- off += align256((size_t) k * 2);- l.hl.svh[r] = file_experts ? l.piece_src[r][2] : off;- off += align256((size_t) n * 2);- }- l.hl.bytes = off;- }- l.hh.assign((size_t) l.n_exp, {nullptr, nullptr, nullptr, nullptr});- if (l.lay.fmt == gg::kFmtMXFP4) { // a GGUF's: read in place from the file's mapping- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = (uint8_t*) mx_mat(l, e, 0);- } else if (file_experts) { // every expert's file mapped (the cache's swaps hand VRAM experts back to the files)- for (int e = 0; e < l.n_exp; ++e)- if (pk.map(l.region_file[(size_t) e]) == nullptr) {- err = "mapping " + pk.files[(size_t) l.region_file[(size_t) e]] + " failed";- return false;- }- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] < 0) {- l.hh[(size_t) e][0] = file_part(l, e);- l.host[(size_t) e] = l.hh[(size_t) e][0];- }- } else if (n_host > 0) {- const int n_parts = n_host + (l.spare_vram >= 0 ? 1 : 0); // (+ the cache's spare)- const size_t bytes = (size_t) n_parts * l.hl.bytes;- for (int X = 0; X < NH; ++X) {- void* m = node_map(bytes, X, NH);- if (m == nullptr) {- err = "host memory for layer " + std::to_string(l.il) + "'s " + std::to_string(n_host) +- " CPU experts (" + std::to_string(bytes >> 20) + " MiB on node " + std::to_string(X) + ")";- return false;- }- host_maps.emplace_back(m, bytes);- int hi = 0;- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] < 0) l.hh[(size_t) e][(size_t) X] = (uint8_t*) m + (size_t) (hi++) * l.hl.bytes;- if (l.spare_vram >= 0) l.spare_host[(size_t) X] = (uint8_t*) m + (size_t) hi * l.hl.bytes;- }- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = l.hh[(size_t) e][0];- }- l.vm.assign((size_t) l.n_exp, {});- if (l.nm == 1) // (the host-routed window path's views: a group's members take the device-routed path)- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] >= 0) set_vm(l, e, l.slot[(size_t) e]);- if (!halloc(l.tab_pin, (size_t) 2 * l.nm, err)) return false;- for (int m = 0; m < l.nm; ++m) {- Layer& x = member(l, m);- std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] >= 0)- tab[(size_t) e] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) e] * unit(x));- cudaSetDevice(devs[(size_t) x.dev].id);- CK(cudaMemcpy(x.vram_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice));- }- }- return true;- }-- // a GGUF's VRAM experts, in file order, by a few threads through pinned staging blobs (the CPU lane's stay in the- // file: the mapping reads them on first use)- bool load_experts_mx(std::string& err) {- struct Job {- int layer, e;- };- std::vector<Job> jobs;- size_t maxb = 0;- for (const Layer& l : L) {- maxb = std::max(maxb, l.blob);- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});- }- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];- if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];- return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];- });- std::atomic<size_t> next{0};- std::atomic<bool> failed{false};- std::mutex emu;- std::string ferr;- std::atomic<int64_t> done_bytes{0};- const double t0 = now_ms();- std::vector<std::thread> th;- for (int t = 0; t < 12; ++t)- th.emplace_back([&] {- uint8_t *stage = nullptr, *stage2 = nullptr;- cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);- cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);- for (;;) {- const size_t i = next.fetch_add(1);- if (i >= jobs.size() || failed.load()) break;- const Layer& l = L[(size_t) jobs[i].layer];- const int e = jobs[i].e;- bool ok = stage != nullptr && stage2 != nullptr && read_mx(l, e, stage);- for (int m = 0; m < l.nm && ok; ++m) { // (a group: each member its part)- const Layer& x = member(l, m);- cudaSetDevice(devs[(size_t) x.dev].id);- if (l.nm > 1) blob_to_half(l, stage, m, stage2);- ok = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(l), l.nm > 1 ? stage2 : stage,- unit(l), cudaMemcpyHostToDevice) == cudaSuccess;- }- if (!ok) {- std::lock_guard<std::mutex> lk(emu);- ferr = "loading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";- failed.store(true);- break;- }- const int64_t db = done_bytes.fetch_add((int64_t) l.blob) + (int64_t) l.blob;- if (o.verbose && (i % 1024) == 0)- std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,- db / 1e6 / std::max(1.0, now_ms() - t0));- }- if (stage) cudaFreeHost(stage);- if (stage2) cudaFreeHost(stage2);- });- for (auto& t : th) t.join();- if (failed.load()) {- err = ferr;- return false;- }- // as load_experts in place: the VRAM experts' pages leave the cache, the most-routed of the others (the- // profile's share of their layer, then by index) are read once, up to warm_bytes, so the cache starts with them- struct Warm {- int layer, e;- double share;- };- std::vector<Warm> warm;- for (const Layer& l : L) {- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;- double sum = 0;- for (int e = 0; e < l.n_exp; ++e) sum += p[e];- for (int e = 0; e < l.n_exp; ++e) {- if (l.slot[(size_t) e] >= 0) {- for (int r = 0; r < 3; ++r)- os::advise(pk.fds[(size_t) l.mx_file[r]], (l.mx_off[r] + (uint64_t) e * l.mx_stride[r]), mx_part_bytes(l, r), false);- } else {- warm.push_back({l.il, e, sum > 0 ? p[e] / sum : 0.0});- }- }- }- std::stable_sort(warm.begin(), warm.end(), [](const Warm& a, const Warm& b) { return a.share > b.share; });- if (!ram_tier && !warm.empty()) { // copied into RAM (copy_experts decides), the rest read ahead into what is left- std::vector<std::pair<int, int>> hot;- for (const Warm& w : warm) hot.push_back({w.layer, w.e});- if (!copy_experts(hot, err)) return false;- if (copy_mode) {- warm.erase(std::remove_if(warm.begin(), warm.end(),- [&](const Warm& w) { return L[(size_t) w.layer].ram[(size_t) w.e] >= 0; }),- warm.end());- const uint64_t av = ram_avail();- if (av != ~uint64_t{0}) warm_bytes = av > kRamMargin ? std::min<uint64_t>(warm_bytes, av - kRamMargin) : 0;- }- }- if (ram_tier && warm_bytes > 0 && !warm.empty()) {- // chunks of 32 slots up to warm_bytes (all the RAM the rest leaves, kRamMargin kept), one slot kept free for- // a move in; the hottest experts read in, their pages then dropped from the page cache. Plain memory, not- // pinned: only the CPU reads it (what goes to a GPU passes the pinned staging), and pinned memory is capped on- // Windows - the engine's own pinned buffers come after this- const size_t want = std::min(warm.size(), (size_t) (warm_bytes / maxb));- while (ram_slots.size() < want + 1) {- const size_t n = std::min<size_t>(32, want + 1 - ram_slots.size());- uint8_t* c = (uint8_t*) os::anon_alloc(n * maxb, -1, 1);- if (c == nullptr) break;- ram_chunks.push_back(c);- ram_chunk_bytes.push_back(n * maxb);- for (size_t i = 0; i < n; ++i) ram_slots.push_back(c + i * maxb);- }- ram_held = ram_slots.empty() ? 0 : (int) ram_slots.size() - 1;- for (Layer& l : L)- if (l.lay.fmt == gg::kFmtMXFP4) l.ram.assign((size_t) l.n_exp, -1);- std::atomic<size_t> ri{0};- std::atomic<bool> rfail{false};- std::vector<std::thread> rt;- for (int t = 0; t < 12 && ram_held > 0; ++t)- rt.emplace_back([&] {- for (size_t i; (i = ri.fetch_add(1)) < (size_t) ram_held;) {- const Layer& l = L[(size_t) warm[i].layer];- if (!read_mx_file(l, warm[i].e, ram_slots[i])) rfail.store(true);- for (int r = 0; r < 3; ++r)- os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) warm[i].e * l.mx_stride[r],- mx_part_bytes(l, r), false);- }- });- for (auto& t : rt) t.join();- if (rfail.load()) {- err = "reading the RAM tier's experts from the model's files failed";- return false;- }- for (int i = 0; i < ram_held; ++i) L[(size_t) warm[(size_t) i].layer].ram[(size_t) warm[(size_t) i].e] = i;- if (ram_held > 0) ram_free.push_back(ram_held);- if (o.expert_cache && ram_held > 0) ram_th = std::thread([this] { ram_loop(); });- if (o.verbose)- std::fprintf(stderr, "ds4: RAM tier: %d experts (%.1f GB of RAM), the hottest after VRAM's; the other %zu "- "read from the model's files (%.1f s)\n",- ram_held, ram_slots.size() * (double) maxb / 1e9, warm.size() - (size_t) ram_held,- (now_ms() - t0) / 1000.0);- return true;- }- size_t nw = 0;- for (double b = 0; nw < warm.size() && b + (double) L[(size_t) warm[nw].layer].blob <= (double) warm_bytes; ++nw)- b += (double) L[(size_t) warm[nw].layer].blob;- std::atomic<size_t> wi{0};- std::vector<std::thread> wt;- for (int t = 0; t < 12 && nw > 0; ++t)- wt.emplace_back([&] {- std::vector<uint8_t> buf(maxb);- for (size_t i; (i = wi.fetch_add(1)) < nw;) read_mx(L[(size_t) warm[i].layer], warm[i].e, buf.data());- });- for (auto& t : wt) t.join();- if (o.verbose && nw > 0)- std::fprintf(stderr, "ds4: %zu CPU-lane experts read ahead (%.1f GB) in %.1f s\n", nw,- nw * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);- return true;- }-- // a GGUF's CPU-lane experts copied into RAM (copy_mode), decided once VRAM's are in - on a unified-memory APU the GPU- // has taken its share of the RAM by then, so the copies only get what it leaves. `hot`: the CPU lane's experts, the- // most-routed first. --mmap-experts off: all that fit beside kRamMargin (a warning if not all do); on and auto:- // none - read in place: the page cache follows a conversation's experts where a static set (the profile's- // hottest) does not (the RAM tier's lesson), and copies that all fit were no faster; DS4_COPY_GB=<n>: the- // hottest n GB (the static hybrid, for tests). Each copy in NH parts on the NUMA nodes (DS4_COPY_NUMA=1: one), read from the files in file- // order by a few threads, anonymous memory on huge pages (Windows: plain), not pinned (only the CPU reads it; what- // goes to a GPU passes the pinned staging) - a layer that cannot get its memory keeps its experts in place- bool copy_experts(const std::vector<std::pair<int, int>>& hot, std::string& err) {- if (o.mmap_experts > 0 || hot.empty()) return true;- const double t0 = now_ms();- auto spare_of = [&](const Layer& l) { return o.expert_cache && l.spare_vram >= 0 ? 1 : 0; };- double all = 0;- {- std::vector<char> seen(L.size(), 0);- for (const auto& h : hot) {- const Layer& l = L[(size_t) h.first];- all += (double) l.blob * (1 + (seen[(size_t) l.il]++ == 0 ? spare_of(l) : 0));- }- }- const uint64_t avail = ram_avail();- // (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)- const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin),- commit_room());- // (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are- // compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed- // on 64 GB and on 32 GB of RAM)- double budget = o.mmap_experts == 0 ? std::min(all, can) : 0.0;- if (const char* v = std::getenv("DS4_COPY_GB")) budget = std::atof(v) * 1e9;- if (o.mmap_experts == 0 && all > can)- std::fprintf(stderr, "ds4: --mmap-experts off: the CPU lane's experts (%.1f GB) do not all fit the RAM this "- "process can get (%.1f GB beside %.0f GB for the rest) - the most-routed copied, the "- "others read in place\n", all / 1e9, can / 1e9, kRamMargin / 1e9);- if (budget <= 0) {- if (o.verbose)- std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) stay in place: the RAM this process can get "- "(%.1f GB beside %.0f GB for the rest) does not hold them all\n",- all / 1e9, can / 1e9, kRamMargin / 1e9);- return true;- }- // NUMA parts: whole rows of each matrix, so any row count the nodes divide- int nh = (int) std::min<size_t>(2, std::max<size_t>(1, cores.size()));- if (const char* v = std::getenv("DS4_COPY_NUMA")) // (0 / 1: one part; 2: two, even on one node - a test)- nh = std::atoi(v) >= 2 ? 2 : 1;- if (g.ff % nh != 0 || g.dim % nh != 0) nh = 1;- NH = nh;- cp_lay.assign(L.size(), {0, 0, 0, 0});- for (const Layer& l : L) {- size_t off = 0;- for (int r = 0; r < 3; ++r) {- cp_lay[(size_t) l.il][(size_t) r] = off;- off += align256((size_t) (mx_n(r) / NH) * mx_row_b(l, r));- }- cp_lay[(size_t) l.il][3] = off;- }- // the hottest that fit the budget (a layer's first one brings its spare)- std::vector<int> n_copy(L.size(), 0);- double used = 0;- size_t take = 0;- for (; take < hot.size(); ++take) {- const Layer& l = L[(size_t) hot[take].first];- const double add = (double) l.blob * (1 + (n_copy[(size_t) l.il] == 0 ? spare_of(l) : 0));- if (used + add > budget) break;- used += add;- ++n_copy[(size_t) l.il];- }- // each layer's slots: one block per node of (copies + spare) parts- std::vector<int> first(L.size(), -1);- copy_spare.assign(L.size(), -1);- for (Layer& l : L) {- l.ram.assign((size_t) l.n_exp, -1);- const int ns = n_copy[(size_t) l.il] + (n_copy[(size_t) l.il] > 0 ? spare_of(l) : 0);- if (ns == 0) continue;- const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;- std::array<uint8_t*, 2> base = {nullptr, nullptr};- bool ok = true;- for (int X = 0; X < NH && ok; ++X) {- base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);- ok = base[(size_t) X] != nullptr;- if (ok) {- ram_chunks.push_back(base[(size_t) X]);- ram_chunk_bytes.push_back(bytes);- }- }- if (!ok) {- std::fprintf(stderr, "ds4: no RAM for layer %d's %d copied experts (%.1f GB): they stay in place\n", l.il,- ns, (double) bytes * NH / 1e9);- n_copy[(size_t) l.il] = 0;- continue;- }- first[(size_t) l.il] = (int) ram_part.size();- for (int i = 0; i < ns; ++i)- ram_part.push_back({base[0] + (size_t) i * part, NH > 1 ? base[1] + (size_t) i * part : nullptr});- if (spare_of(l)) copy_spare[(size_t) l.il] = first[(size_t) l.il] + ns - 1;- }- struct Job {- int layer, e, slot;- };- std::vector<Job> jobs;- {- std::vector<int> next = first;- for (size_t i = 0; i < take; ++i) {- const int il = hot[i].first;- if (n_copy[(size_t) il] == 0) continue;- jobs.push_back({il, hot[i].second, next[(size_t) il]++});- }- }- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) { // (file order: the disk reads ahead)- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];- if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];- return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];- });- std::atomic<size_t> ji{0};- std::atomic<bool> bad{false};- std::atomic<int64_t> done_b{0};- std::vector<std::thread> th;- for (int t = 0; t < 12 && !jobs.empty(); ++t)- th.emplace_back([&] {- for (size_t i; (i = ji.fetch_add(1)) < jobs.size() && !bad.load();) {- const Job& j = jobs[i];- const Layer& l = L[(size_t) j.layer];- for (int r = 0; r < 3; ++r) {- const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);- for (int X = 0; X < NH; ++X)- if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) j.e * l.mx_stride[r] + (uint64_t) X * nb, nb,- ram_part[(size_t) j.slot][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r]))- bad.store(true);- }- const int64_t db = done_b.fetch_add((int64_t) l.blob) + (int64_t) l.blob;- if (o.verbose && i % 2048 == 0)- std::fprintf(stderr, "ds4: copying experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,- db / 1e6 / std::max(1.0, now_ms() - t0));- }- });- for (auto& t : th) t.join();- if (bad.load()) {- err = "copying the CPU lane's experts from the model's files failed";- return false;- }- for (const Job& j : jobs) L[(size_t) j.layer].ram[(size_t) j.e] = j.slot;- copy_mode = true;- ram_held = (int) jobs.size();- double cb = 0;- for (const Job& j : jobs) cb += (double) L[(size_t) j.layer].blob;- std::fprintf(stderr, "ds4: %zu of the CPU lane's %zu experts copied into RAM (%.1f GB, %d NUMA part%s) in %.1f s%s\n",- jobs.size(), hot.size(), cb / 1e9, NH, NH > 1 ? "s" : "", (now_ms() - t0) / 1000.0,- jobs.size() < hot.size() ? "; the others read in place" : "");- return true;- }-- bool load_experts(std::string& err) {- if (!L.empty() && L[0].lay.fmt == gg::kFmtMXFP4) return load_experts_mx(err);- // every expert's region, in file order, by a few threads; VRAM ones through a pinned staging blob. In place- // (file_experts) the CPU lane's experts are not copied: the most-routed of them (the profile's share of their- // layer) are read once, up to warm_bytes, so the page cache starts with them, and the VRAM ones' pages are- // dropped from it again- struct Job {- int layer, e;- };- std::vector<Job> jobs;- for (const Layer& l : L)- for (int e = 0; e < l.n_exp; ++e)- if (!file_experts || l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});- uint64_t warm = 0;- if (file_experts) {- std::vector<std::pair<double, Job>> cand;- for (const Layer& l : L) {- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;- double sum = 0;- for (int e = 0; e < l.n_exp; ++e) sum += p[e];- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] < 0) cand.push_back({sum > 0 ? p[e] / sum : 0.0, Job{l.il, e}});- }- std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });- for (const auto& c : cand) {- const uint64_t b = L[(size_t) c.second.layer].region_bytes[(size_t) c.second.e];- if (warm + b > warm_bytes) break;- warm += b;- jobs.push_back(c.second);- }- }- std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {- const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];- if (la.region_file[(size_t) a.e] != lb.region_file[(size_t) b.e])- return la.region_file[(size_t) a.e] < lb.region_file[(size_t) b.e];- return la.region_off[(size_t) a.e] < lb.region_off[(size_t) b.e];- });- std::atomic<size_t> next{0};- std::atomic<bool> failed{false};- std::mutex emu;- std::string ferr;- const int nth = 12;- size_t maxr = 0, maxb = 0;- for (const Layer& l : L) {- maxb = std::max(maxb, l.blob);- for (uint64_t r : l.region_bytes) maxr = std::max<size_t>(maxr, r);- }- std::atomic<int64_t> done_bytes{0};- const double t0 = now_ms();- std::vector<std::thread> th;- for (int t = 0; t < nth; ++t)- th.emplace_back([&] {- std::vector<uint8_t> region(maxr);- uint8_t *stage = nullptr, *stage2 = nullptr;- cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);- cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);- int cur_dev = -1;- for (;;) {- const size_t i = next.fetch_add(1);- if (i >= jobs.size() || failed.load()) break;- const Layer& l = L[(size_t) jobs[i].layer];- const int e = jobs[i].e;- if (!pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], l.region_bytes[(size_t) e],- region.data())) {- std::lock_guard<std::mutex> lk(emu);- ferr = "reading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";- failed.store(true);- break;- }- if (l.slot[(size_t) e] >= 0) {- repack(l, region.data(), stage);- bool up = true;- for (int m = 0; m < l.nm && up; ++m) { // (a group: each member its part)- const Layer& x = member(l, m);- const Dev& d = devs[(size_t) x.dev];- if (cur_dev != d.id) {- cudaSetDevice(d.id);- cur_dev = d.id;- }- if (l.nm > 1) blob_to_half(l, stage, m, stage2);- up = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(x), l.nm > 1 ? stage2 : stage,- unit(x), cudaMemcpyHostToDevice) == cudaSuccess;- }- if (!up) {- std::lock_guard<std::mutex> lk(emu);- ferr = "uploading layer " + std::to_string(l.il) + " expert " + std::to_string(e);- failed.store(true);- break;- }- if (file_experts)- os::advise(pk.fds[(size_t) l.region_file[(size_t) e]], l.region_off[(size_t) e], l.region_bytes[(size_t) e], false);- } else if (!file_experts) {- for (int X = 0; X < NH; ++X) repack_part(l, region.data(), X, l.hh[(size_t) e][(size_t) X]);- }- const int64_t db = done_bytes.fetch_add((int64_t) l.region_bytes[(size_t) e]) +- (int64_t) l.region_bytes[(size_t) e];- if (o.verbose && (i % 1024) == 0)- std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(),- db / 1e9, db / 1e6 / std::max(1.0, now_ms() - t0));- }- if (stage) cudaFreeHost(stage);- if (stage2) cudaFreeHost(stage2);- });- for (auto& t : th) t.join();- if (failed.load()) {- err = ferr;- return false;- }- if (file_experts)- std::fprintf(stderr, "ds4: experts loaded in %.1f s (%.1f GB of the CPU lane's read ahead)\n",- (now_ms() - t0) / 1000.0, warm / 1e9);- return true;- }-- std::string describe() const {- std::ostringstream s;- for (size_t di = 0; di < devs.size(); ++di) {- const Dev& d = devs[di];- int a = -1, b = -1, ev = 0, et = 0;- for (const Layer& l : L)- for (int m = 0; m < l.nm; ++m)- if (member(l, m).dev == (int) di) {- if (a < 0) a = l.il;- b = l.il;- ev += l.n_vram;- et += l.n_exp;- }- size_t fr = 0, tot = 0;- cudaSetDevice(d.id);- cudaMemGetInfo(&fr, &tot);- const size_t gsz = groups.empty() ? 1 : groups[(size_t) d.grp].mem.size();- char tp[64] = "";- if (gsz > 1) std::snprintf(tp, sizeof tp, " (tensor parallel %zu, part %d)", gsz, d.mi);- char line[384];- std::snprintf(line, sizeof line,- "ds4: CUDA%d layers [%d, %d]%s%s: weights %.2f GB, experts %.2f GB (%d of %d in VRAM), caches "- "%.2f GB, %.0f MiB free\n",- d.id, a, b, tp, (int) di == head_dev ? " + head" : "", d.weights / 1e9, d.experts / 1e9, ev, et,- d.caches / 1e9, fr / 1048576.0);- s << line;- }- int nh = 0;- size_t hb_ = 0;- for (const Layer& l : L) {- nh += l.n_exp - l.n_vram;- hb_ += (size_t) (l.n_exp - l.n_vram) * l.blob;- }- if (std::getenv("DS4_PLACE_HASH")) { // debugging: which experts hold a VRAM slot, as one hash (is the placement the same from run to run?)- uint64_t h = 1469598103934665603ull;- for (const Layer& l : L)- for (int e = 0; e < l.n_exp; ++e) { h ^= (uint64_t) (l.slot[(size_t) e] >= 0 ? 1 + l.il * 1000 + e : 0); h *= 1099511628211ull; }- char hl[96];- std::snprintf(hl, sizeof hl, "ds4: placement hash %016llx\n", (unsigned long long) h);- s << hl;- }- char line[200];- if (copy_mode)- std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB): %d copied into RAM (%d NUMA "- "parts), the rest in place in the model's files; %d threads\n",- nh, hb_ / 1e9, ram_held, NH, cpu_threads());- else if (file_experts)- std::snprintf(line, sizeof line,- "ds4: %d routed experts on the CPU lane (%.1f GB, in place in the model's files), %d threads\n",- nh, hb_ / 1e9, cpu_threads());- else- std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB of RAM, %d NUMA parts), %d threads\n", nh,- hb_ / 1e9, NH, cpu_threads());- s << line;- return s.str();- }-- double store_gb() const {- static double cached = -2.0; // (decided once: the layout of the blobs depends on it)- if (cached > -1.5) return cached;- double gb = o.ssd_cache_gb;- if (const char* v = std::getenv("DS4_STORE_GB")) gb = std::atof(v);- if (!pk.gguf) gb = 0.0;- else if (gb < 0.0) { // auto: the RAM free now less 12 GB for the system and the owner's programs, 48 GB at most- const uint64_t avail = ram_avail();- const double room = avail == ~uint64_t{0} ? 0.0 : (double) avail / 1e9 - 12.0;- gb = room >= 4.0 ? std::min(48.0, room) : 0.0;- }- return cached = gb;- }- void init_store() {- const double gb = store_gb();- if (gb <= 0.0 || L.empty() || L[0].lay.fmt != gg::kFmtMXFP4 || !file_experts) return;- std::vector<ExpertStore::LayerGeo> geo((size_t) g.nl); // (the trunk's layers: the drafter's bigger experts stay in place)- for (const Layer& l : L) {- if (l.il >= g.nl) continue;- ExpertStore::LayerGeo& gl = geo[(size_t) l.il];- gl.n_exp = l.n_exp;- for (int r = 0; r < 3; ++r) {- gl.file[r] = l.mx_file[r];- gl.off[r] = l.mx_off[r];- gl.stride[r] = l.mx_stride[r];- gl.bytes[r] = mx_part_bytes(l, r);- gl.tr[r] = l.lay.trellis[r];- }- gl.blob = l.blob;- }- ExpertStore::Cfg cfg;- cfg.budget_bytes = (size_t) (gb * 1e9);- cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;- cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;- cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);- cfg.ram_avail = [] { return ram_avail(); };- cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);- cfg.vis_percent = std::getenv("DS4_STORE_ZC_FRAC") ? std::atoi(std::getenv("DS4_STORE_ZC_FRAC")) : 50;- // (the GPU computing the store's experts over PCIe is opt-in, DS4_STORE_ZC=1: measured slower than the CPU lane alone on a- // 5070 / PCIe 5.0 - the staging copy sits in the layer's critical path - and its answers depend on the timing of which lane- // took which expert, the CPU lane's are the mmap path's byte for byte)- const bool want_zc = std::getenv("DS4_STORE_ZC") && std::atoi(std::getenv("DS4_STORE_ZC")) != 0 && devs.size() == 1 &&- L[0].nm == 1;- if (want_zc) { // the arena's chunks registered for the GPU to read in place (zero-copy over PCIe)- const int dev0 = devs[0].id;- cfg.map_chunk = [dev0](void* h, size_t n) -> void* {- cudaSetDevice(dev0);- if (cudaHostRegister(h, n, cudaHostRegisterMapped | cudaHostRegisterPortable) != cudaSuccess) {- (void) cudaGetLastError();- return nullptr;- }- void* d = nullptr;- if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) {- (void) cudaGetLastError();- cudaHostUnregister(h);- return nullptr;- }- if (d != h) std::fprintf(stderr, "ds4: RAM store: chunk host %p -> device %p (not the identity)\n", h, d);- return d;- };- cfg.unmap_chunk = [dev0](void* h) {- cudaSetDevice(dev0);- cudaHostUnregister(h);- };- }- bool direct = true;- for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;- auto st = std::make_unique<ExpertStore>();- if (!st->init(cfg, geo, pk.dfds, pk.pfds)) {- std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");- return;- }- store = std::move(st);- for (Layer& l : L)- if (l.il < g.nl && want_zc) l.zc.assign((size_t) l.n_exp, 0);- std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",- store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,- (int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");- if (want_zc) {- if (!store->gpu_mapped()) {- std::fprintf(stderr, "ds4: RAM store: the arena could not be mapped for the GPU: zero-copy off\n");- for (Layer& l : L) l.zc.clear();- } else {- zc_stride = (store->max_blob() + 4095) & ~(size_t) 4095;- cudaSetDevice(devs[0].id);- if (cudaMalloc((void**) &zc_stage_buf, zc_stride * kPicks) != cudaSuccess) {- (void) cudaGetLastError();- zc_stage_buf = nullptr;- for (Layer& l : L) l.zc.clear();- } else {- for (Layer& l : L)- if (!l.zc.empty() && cudaMalloc((void**) &l.zc_tab, (size_t) l.n_exp * 8) == cudaSuccess) cudaMemset(l.zc_tab, 0, (size_t) l.n_exp * 8);- else if (!l.zc.empty()) {- (void) cudaGetLastError();- l.zc.clear();- }- }- }- std::fprintf(stderr, "ds4: RAM store: the GPU takes its slots over PCIe (%s), %zu slots kept evictable\n",- zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);- }- }- // the profile's hottest experts (VRAM's left out) read into the store before the first token (DS4_STORE_PRELOAD=0: not)- void preload_store() {- if (!store) return;- if (std::getenv("DS4_STORE_PRELOAD") && std::atoi(std::getenv("DS4_STORE_PRELOAD")) == 0) return;- std::vector<std::pair<double, std::pair<int, int>>> cand;- double tot = 0;- for (const Layer& l : L) {- if (l.il >= g.nl) continue;- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;- for (int e = 0; e < l.n_exp; ++e) tot += p[e];- }- if (tot <= 0) return; // (no profile yet: the first tokens fill it)- for (const Layer& l : L) {- if (l.il >= g.nl) continue;- const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] < 0 && p[e] > 0) cand.push_back({(double) p[e] / (double) l.blob, {l.il, e}});- }- std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });- std::vector<std::pair<int, int>> list;- list.reserve(cand.size());- for (const auto& c : cand) list.push_back(c.second);- const double t0 = now_ms();- const size_t leave = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;- const size_t n = store->preload(list, leave);- std::fprintf(stderr, "ds4: RAM store: %zu of the profile's hottest experts read in (%.1f GB) in %.1f s\n", n,- (double) n * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);- }- // at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;- // each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window- void store_sync() {- if (!store || !store->gpu_mapped()) return;- std::vector<ExpertStore::Vis> ev;- store->sync(ev, [&](int layer, int e) { return L[(size_t) layer].slot[(size_t) e] < 0; });- if (ev.empty()) return;- std::vector<char> dirty(L.size(), 0);- for (const auto& v : ev) {- Layer& l = L[(size_t) v.layer];- if (l.zc.empty()) continue;- l.zc[(size_t) v.e] = v.dev != 0 ? 1 : 0;- dirty[(size_t) v.layer] = 1;- }- for (Layer& l : L) {- if (!dirty[(size_t) l.il]) continue;- if (l.zc_tab == nullptr) continue;- std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);- for (int e = 0; e < l.n_exp; ++e)- if (l.zc[(size_t) e]) {- const int s = l.slot[(size_t) e] < 0 ? store->find(l.il, e) : -1;- tab[(size_t) e] = s >= 0 ? store->dev_blob(s) : 0ull;- if (s < 0) l.zc[(size_t) e] = 0;- }- cudaSetDevice(devs[0].id);- cudaMemcpyAsync(l.zc_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice, devs[0].st);- }- }-- int cpu_threads() const {- int n = 0;- for (const auto& pl : pools) n += pl->size() - 1;- return n + 1;- }- // the CPU lane's workers: a pool per NUMA node (NH of them) on its cores; in place with one part, one pool over- // every node's cores (the pages lie where the cache put them)- void make_pools() {- std::vector<std::vector<int>> cs = cores;- if ((int) cs.size() < NH && !cs.empty()) { // (DS4_COPY_NUMA=2 on one node, a test: its cores split in NH)- std::vector<int> all = cs[0];- cs.assign((size_t) NH, {});- for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);- }- for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {- std::vector<int> c = cs[(size_t) X];- if (file_experts && NH == 1)- for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());- if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)- int nw = (int) c.size();- if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));- pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)- }- (void) gg::mx_tier(); // (measured here, not on the first token)- if (o.verbose) {- int nthr = 0;- for (auto& p : pools) nthr += p->size();- std::fprintf(stderr, "ds4: CPU lane: %d threads, MXFP4 rows %s (%s)\n", nthr, gg::mx_tier_name(gg::mx_tier()),- gg::mx_tier_report());- }- }-- void reset() {- pos = 0;- toks.clear();- ctoks.clear();- ck[0].pos = ck[1].pos = -1;- for (Layer& l0 : L)- for (int m = 0; m < l0.nm; ++m) {- Layer& l = member(l0, m);- Dev& d = devs[(size_t) l.dev];- cudaSetDevice(d.id);- cudaMemsetAsync(l.win, 0, (size_t) g.win * win_rb(), d.st);- if (l.c_kv) cudaMemsetAsync(l.c_kv, 0, (size_t) l.ratio * g.hd * 4, d.st);- if (l.c_score) cudaMemsetAsync(l.c_score, 0, (size_t) l.ratio * g.hd * 4, d.st);- }- for (Dev& d : devs) {- cudaSetDevice(d.id);- cudaStreamSynchronize(d.st);- }- }-- // the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the- // state is indexed by position, and what lies past a restored position is written again)- void checkpoint(int slot) {- Check& c = ck[slot & 1];- c.pos = pos;- c.win.resize(L.size());- c.comp.resize(L.size() * 2);- for (size_t i = 0; i < L.size(); ++i) {- Layer& l = L[i];- Dev& d = devs[(size_t) l.dev];- cudaSetDevice(d.id);- c.win[i].resize((size_t) g.win * win_rb());- cudaMemcpyAsync(c.win[i].data(), l.win, c.win[i].size(), cudaMemcpyDeviceToHost, d.st);- if (l.c_kv) {- c.comp[2 * i].resize((size_t) l.ratio * g.hd);- c.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);- cudaMemcpyAsync(c.comp[2 * i].data(), l.c_kv, c.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, d.st);- cudaMemcpyAsync(c.comp[2 * i + 1].data(), l.c_score, c.comp[2 * i + 1].size() * 4,- cudaMemcpyDeviceToHost, d.st);- }- }- for (Dev& d : devs) {- cudaSetDevice(d.id);- cudaStreamSynchronize(d.st);- }- }-- // the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1- int best_check(int64_t limit) const {- int b = -1;- for (int i = 0; i < 2; ++i)- if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;- return b;- }- int64_t restorable(int64_t limit) const {- const int b = best_check(limit);- return b < 0 ? -1 : ck[b].pos;- }-- bool restore(int64_t limit) {- const int b = best_check(limit);- if (b < 0) return false;- const Check& c = ck[b];- for (size_t i = 0; i < L.size(); ++i)- for (int m = 0; m < L[i].nm; ++m) { // (a group's members hold the same state: the lead's copy)- Layer& l = member(L[i], m);- Dev& d = devs[(size_t) l.dev];- cudaSetDevice(d.id);- cudaMemcpyAsync(l.win, c.win[i].data(), c.win[i].size(), cudaMemcpyHostToDevice, d.st);- if (l.c_kv) {- cudaMemcpyAsync(l.c_kv, c.comp[2 * i].data(), c.comp[2 * i].size() * 4, cudaMemcpyHostToDevice, d.st);- cudaMemcpyAsync(l.c_score, c.comp[2 * i + 1].data(), c.comp[2 * i + 1].size() * 4,- cudaMemcpyHostToDevice, d.st);- }- }- for (Dev& d : devs) {- cudaSetDevice(d.id);- cudaStreamSynchronize(d.st);- }- pos = c.pos;- toks.resize((size_t) pos);- ctoks.resize((size_t) pos);- for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has- if (o2.pos > pos) o2.pos = -1;- return true;- }-- size_t bounce_bytes() const {- return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);- }- size_t topk_stride() const { return (size_t) std::max(g.ix_topk, 32); }- size_t cand_stride() const { return (size_t) std::max(g.cand_topk, 32); }- size_t bm_stride() const { return (size_t) (o.max_context + 64) / std::max(1, g.cand_blk) + 64; }- static constexpr int kSlots = kWin * kPicks; // the CPU lane's (token, expert) rows a window can need-- // ---- the vision tower (the checkpoint's vision.* / aligner.*) on the first GPU, its weights streamed from the- // host; a prompt's pictures as their spans' rows (set_images)- VisionTower vt;- VisionGeo vg;- int img_tok = -1;- std::string vision_err; // why the tower did not load (pictures off), or empty- std::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]- std::vector<Image> imgs; // the pictures of the prompt being fed, by start- // a BF16 (or, rounded to nearest even, F32) tensor of n values into dst- bool read_bf16(const Tensor& t, size_t n, void* dst) {- if (t.dtype == "BF16") return t.bytes == n * 2 && pk.read(t, dst);- if (t.dtype != "F32" || t.bytes != n * 4) return false;- std::vector<float> f(n);- if (!pk.read(t, f.data())) return false;- uint16_t* o = (uint16_t*) dst;- for (size_t i = 0; i < n; ++i) {- uint32_t u;- std::memcpy(&u, &f[i], 4);- o[i] = (uint16_t) ((u + 0x7FFFu + ((u >> 16) & 1u)) >> 16);- }- return true;- }- bool load_vision(std::string& err) {- vg.layers = (int) pk.num("vision.num_hidden_layers", 0);- const int tok = (int) pk.num("image_token_id", -1);- if (vg.layers <= 0 || tok < 0 || pk.get("vision.patch_embed.proj.weight") == nullptr) return true; // none- vg.dim = (int) pk.num("vision.hidden_size", 1024), vg.heads = (int) pk.num("vision.num_attention_heads", 16);- vg.inter = (int) pk.num("vision.intermediate_size", 2816), vg.patch = (int) pk.num("vision.patch_size", 14);- vg.merge = (int) pk.num("vision.downsample_ratio", 3), vg.theta = (float) pk.num("vision.rope_theta", 1e4);- vg.max_tokens = (int) pk.num("vision.max_image_tokens", 1024), vg.out = g.dim;- const char* mark[3] = {"image_start", "image_newline", "image_end"};- for (int i = 0; i < 3; ++i) {- const Tensor* t = pk.get(mark[i]);- img_mark[i].resize((size_t) g.dim);- if (t == nullptr || !read_bf16(*t, (size_t) g.dim, img_mark[i].data())) {- err = std::string(mark[i]) + ": not a BF16 (or F32) row of the model's width";- return false;- }- }- const double t0 = now_ms();- Dev& d = devs[0];- // the tower's tensors as BF16: the checkpoint's own, or a GGUF's mmproj (F32 norms / biases / marker rows, which- // came from BF16, converted back exactly; mlp.w1 as the two files' gate and up, the halves of its rows)- auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {- const Tensor* t = pk.get(n);- if (t == nullptr && n.size() > 13 && n.compare(n.size() - 13, 13, "mlp.w1.weight") == 0) {- const std::string pre = n.substr(0, n.size() - 13);- const Tensor* tg = pk.get(pre + "mlp.gate.weight");- const Tensor* tu = pk.get(pre + "mlp.up.weight");- if (tg == nullptr || tu == nullptr || tg->shape.size() != 2 || tg->shape != tu->shape) return false;- shape = {tg->shape[0] * 2, tg->shape[1]};- const size_t half = (size_t) tg->shape[0] * (size_t) tg->shape[1];- return dst == nullptr || (read_bf16(*tg, half, dst) && read_bf16(*tu, half, (uint16_t*) dst + half));- }- if (t == nullptr) return false;- shape = t->shape;- size_t n_el = 1;- for (int64_t x : t->shape) n_el *= (size_t) x;- return dst == nullptr || read_bf16(*t, n_el, dst);- };- if (!vt.load(vg, d.id, read, err)) return false;- img_tok = tok;- CK(cudaSetDevice(d.id));- CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)- if (o.verbose)- std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",- vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),- (now_ms() - t0) / 1000.0);- return true;- }- bool encode_image(const uint8_t* rgb, int w, int h, std::vector<uint16_t>& span, std::string& err) {- if (img_tok < 0) {- err = "this model has no vision tower (or it is off)";- return false;- }- std::vector<uint16_t> rows;- int nh = 0, nw = 0;- if (!vt.encode(rgb, w, h, rows, nh, nw, err)) return false;- const size_t D = (size_t) g.dim;- span.resize(((size_t) nh * (nw + 1) + 2) * D);- uint16_t* o = span.data();- auto put = [&](const uint16_t* r) {- std::memcpy(o, r, D * 2);- o += D;- };- put(img_mark[0].data());- for (int i = 0; i < nh; ++i) {- for (int j = 0; j < nw; ++j) put(rows.data() + ((size_t) i * nw + j) * D);- put(img_mark[1].data());- }- put(img_mark[2].data());- return true;- }- // the row of a picture's span at position p (null: p is in none)- const uint16_t* image_row(int64_t p) const {- if (imgs.empty()) return nullptr;- auto it = std::upper_bound(imgs.begin(), imgs.end(), p, [](int64_t v, const Image& im) { return v < im.start; });- if (it == imgs.begin()) return nullptr;- const Image& im = *(it - 1);- const int64_t r = p - im.start;- if (!im.rows || r >= (int64_t) (im.rows->size() / (size_t) g.dim)) return nullptr;- return im.rows->data() + (size_t) r * g.dim;- }- // token tk's embedding row at position p: inside a picture's span, the picture's- const uint16_t* emb_row(int64_t p, int tk) const {- const uint16_t* r = tk == img_tok ? image_row(p) : nullptr;- return r != nullptr ? r : embed_pinned + (size_t) tk * g.dim;- }- bool in_image(int64_t p) const { return img_tok >= 0 && toks[(size_t) p] == img_tok && image_row(p) != nullptr; }- // the Engram's compressed id of token tk at p (-1: a picture's, which no n-gram spans)- int32_t ctok(int64_t p, int tk) const {- if (tk == img_tok && image_row(p) != nullptr) return -1;- return tk < pk.eg_vocab ? pk.eg_map[(size_t) tk] : 0;- }- // a chunk's image rows (route()'s bits); false: none- bool vl_bits(int64_t p0, int T, VlBits& b) const {- if (imgs.empty()) return false;- std::memset(&b, 0, sizeof b);- bool any = false;- for (int t = 0; t < T && t < kVlRows; ++t)- if (in_image(p0 + t)) {- b.w[t >> 5] |= 1u << (t & 31);- any = true;- }- return any;- }-- // ---- Engram: a window's rows of one table (Engram layer li), tokens p0 .. p0 + T - 1 into dst ([token] of [cols]- // [hd] codes then the scales; nullptr: h_eg). eg_issue serves the rows already fetched from the row cache (and a- // row the window needs twice from one read) and hands the rest to the reader's threads; eg_finish waits for them,- // copies the repeats and keeps the new rows in the cache. The rows are the file's bytes either way.- void eg_issue(int64_t p0, int T, int li, uint8_t* dst, EgBatch& b) {- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, hd = pk.eg_hd;- const size_t per = (size_t) cols * (hd + hd / 32);- b.clear();- b.reads.reserve((size_t) T * cols);- const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";- const Tensor* tw = pk.get(P + "weight");- const Tensor* ts = pk.get(P + "scale");- const int fmt = eg_fmt[(size_t) li];- const bool mx8 = fmt == 0; // (a GGUF: the scales between the codes, one per 32)- const bool q6k = fmt == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)- const bool iq4 = fmt == 2; // (a GGUF: 18-byte blocks a 32 values, d and 16 nibble bytes, no scales)- std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read- int64_t hits = 0;- {- std::lock_guard<std::mutex> lk(eg_cache_mu);- for (int t = 0; t < T; ++t) {- const int64_t p = p0 + t;- uint8_t* rows = dst != nullptr ? dst + (size_t) t * per : h_eg + ((size_t) li * eg_cap + t) * per;- uint8_t* scales = rows + (size_t) cols * hd;- if (ctoks[(size_t) p] < 0) { // a picture's token: no rows (zeros: its gate adds nothing)- std::memset(rows, 0, per);- continue;- }- int64_t prod[8];- bool blocked = false;- for (int s = 0; s < pk.eg_ngram; ++s) { // (the look-back stops at the start and at a picture)- blocked = blocked || p < s || ctoks[(size_t) (p - s)] < 0;- const int64_t tk = blocked ? pk.eg_pad : ctoks[(size_t) (p - s)];- prod[s] = tk * pk.eg_mult[(size_t) li * pk.eg_ngram + s];- }- int64_t rolling = prod[0];- for (int n = 1; n < pk.eg_ngram; ++n) {- rolling ^= prod[n];- for (int h = 0; h < pk.eg_heads; ++h) {- const int c = (n - 1) * pk.eg_heads + h;- const int64_t id = rolling % pk.eg_primes[(size_t) li * cols + c] + pk.eg_off[(size_t) li * cols + c];- uint8_t* rd = rows + (size_t) c * hd;- uint8_t* sd = scales + (size_t) c * (hd / 32);- const uint64_t key = (uint64_t) id * 8 + (uint64_t) li;- if (const uint8_t* hit = eg_cache.find(key)) {- std::memcpy(rd, hit, (size_t) hd);- std::memcpy(sd, hit + hd, (size_t) hd / 32);- ++hits;- continue;- }- const auto fi = first.find(key);- if (fi != first.end()) {- b.dups.push_back({fi->second, {rd, sd}});- continue;- }- first.emplace(key, (int) b.reads.size());- EgRead r;- r.file = tw->file;- r.dst = rd;- r.scales = sd;- r.key = key;- if (mx8) {- r.mx8 = true;- r.n = (uint32_t) (hd / 32 * 33);- r.off = tw->off + (uint64_t) id * r.n;- } else if (q6k || iq4) { // (into the row's code bytes; its scale bytes unused)- r.n = (uint32_t) (q6k ? hd / 256 * 210 : hd / 32 * 18);- r.off = tw->off + (uint64_t) id * r.n;- } else {- r.n = (uint32_t) hd;- r.off = tw->off + (uint64_t) id * hd;- r.file2 = ts->file;- r.n2 = (uint32_t) (hd / 32);- r.off2 = ts->off + (uint64_t) id * (hd / 32);- }- b.reads.push_back(r);- }- }- }- }- eg_rows_n += hits + (int64_t) b.reads.size() + (int64_t) b.dups.size();- eg_hits_n += hits + (int64_t) b.dups.size();- eg_reads_n += (int64_t) b.reads.size();- eg_io.submit(b);- }- // b's rows all in place (any thread; a group's members may ask at once: one finishes, the others wait for it)- void eg_finish(EgBatch& b) {- if (b.post.load(std::memory_order_acquire) == 2) return;- const int64_t t0 = now_ns();- eg_io.wait(b);- int want = 0;- if (b.post.compare_exchange_strong(want, 1, std::memory_order_acq_rel)) {- const int hd = pk.eg_hd;- for (const auto& d : b.dups) {- const EgRead& r = b.reads[(size_t) d.first];- std::memcpy(d.second.first, r.dst, (size_t) hd);- std::memcpy(d.second.second, r.scales, (size_t) hd / 32);- }- if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite- for (size_t i = 0; i < b.reads.size(); ++i) {- const EgRead& r = b.reads[i];- for (int k = 0; k < hd / 32; ++k) {- uint16_t hb;- std::memcpy(&hb, r.dst + (size_t) k * 18, 2);- if ((hb & 0x7c00) == 0x7c00)- std::fprintf(stderr, "DBG eg bad scale: read %zu/%zu file=%d off=%llu n=%u block=%d half=%04x ok=%d\n", i, b.reads.size(),- r.file, (unsigned long long) r.off, r.n, k, hb, (int) b.ok[i]);- }- }- }- {- std::lock_guard<std::mutex> lk(eg_cache_mu);- for (size_t i = 0; i < b.reads.size(); ++i) {- const EgRead& r = b.reads[i];- if (!b.ok[i]) { // (a failed read: zeros, its gate adds nothing; not kept)- std::memset(r.dst, 0, (size_t) hd);- std::memset(r.scales, 0, (size_t) hd / 32);- continue;- }- if (uint8_t* c = eg_cache.put(r.key)) {- std::memcpy(c, r.dst, (size_t) hd);- std::memcpy(c + hd, r.scales, (size_t) hd / 32);- }- }- }- eg_io_ns += b.t1.load() - b.t0;- b.live = false;- b.post.store(2, std::memory_order_release);- } else {- while (b.post.load(std::memory_order_acquire) != 2) std::this_thread::yield();- }- eg_wait_ns += now_ns() - t0;- }- void eg_stats() {- st.eg_rows = eg_rows_n.load();- st.eg_hits = eg_hits_n.load();- st.eg_reads = eg_reads_n.load();- st.eg_io_ms = (double) eg_io_ns.load() / 1e6;- std::lock_guard<std::mutex> lk(eg_cache_mu);- st.eg_cached = (int64_t) eg_cache.size();- st.eg_cache_cap = (int64_t) eg_cache.capacity();- }- // the chunked prompt path: rows of Engram layer `only` into dst, now (a helper thread, a chunk ahead)- void engram_rows(int64_t p0, int T, int only, uint8_t* dst) {- EgBatch b;- eg_issue(p0, T, only, dst, b);- eg_finish(b);- }-- // ---- the CPU lane: a window's routed experts that live on the host, as entries (an expert and up to 8 of its- // tokens, in rows slot0 ..) computed together: each node its column part of every matrix, from memory on that node,- // in four rounds (the gate / up split-K pieces, each preparing its two input blocks of every token; each band's- // sums / codebook / Hadamard / swiglu and that block of the down input, which both nodes then read whole; the down- // pieces; the down bands). yd[slot] gets each row's expert output.- struct Ent {- int e = 0, n = 0, slot0 = 0;- int tok[kc::kExl3CpuMaxN] = {};- };- void run_nodes(int n, const std::function<void(int)>* f) {- for (int X = 0; X < NH; ++X) pools[(size_t) X]->start(n, f[X]);- pools[0]->help();- for (int X = 0; X < NH; ++X) pools[(size_t) X]->wait();- }-- // a GGUF's MXFP4 experts on the CPU lane, read from the file's mapping or their copy in RAM (mx_part_ptr): each- // node its rows of every matrix - the gate / up rows [X n_ff / NH, ..) of each entry (the swiglu into its h row),- // then (h whole) the down rows [X dim / NH, ..) into yd. A row's sum is the same whatever the split: the same bits- // in a copy as in place. The device's arithmetic (gg::mx_rows: int8 activations rounded as the device rounds- // them, exact integer sums a block; every CPU tier the same bits): each token's input quantized once, whatever- // number of entries take it, and each job its band of h straight into the down's int8 input.- // (ord: the order the entries are computed in - their slots, so the sums, stay the same; null: as they are)- void cpu_experts_mx(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {- if (!gg::mx_experts(l.lay)) return cpu_experts_iq(l, x, ents, ldx, ord);- constexpr int RB = 64; // rows a job (whole blocks of h: RB % 32 == 0)- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;- const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();- const size_t gr = gg::mx_row_bytes(E), dr = gg::mx_row_bytes(FF);- const size_t xa = gg::mx_act_bytes(E), ha = gg::mx_act_bytes(FF);- const bool band_q = FFh % 32 == 0; // (each job's rows whole blocks of h)- const float lim = g.lim;- static_assert(kc::kExl3CpuMaxN <= gg::kMxCpuMaxN, "a CPU-lane entry's tokens in one call");- { // the experts read from the files (not in the RAM tier) that are not resident: all their pages asked for at- // once, so the SSD reads them with large requests side by side while the first entries compute- // (os::prefetch_cold: the resident ones are left alone - the hint walks every page of a range, which cost- // more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)- std::vector<std::pair<const void*, size_t>> want;- for (const Ent& en : ents)- if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))- for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});- os::prefetch_cold(want);- }- // the int8 inputs in c_xh's bytes: token t's (t < kWin) at t xa, then the slots' h at (kWin + slot) ha; a token- // index past the window (none on the decode or chunk paths) gets a row of its own after them- uint8_t* xq = (uint8_t*) c_xh.data();- uint8_t* hq = xq + (size_t) kWin * xa;- uint8_t* xq_extra = hq + (size_t) kSlots * ha;- auto x_act = [&](const Ent& en, int i) -> const void* {- const int t = en.tok[i];- return t < kWin ? xq + (size_t) t * xa : xq_extra + (size_t) (en.slot0 + i) * xa;- };- {- bool done[kWin] = {};- for (const Ent& en : ents)- for (int i = 0; i < en.n; ++i) {- const int t = en.tok[i];- if (t < kWin && done[t]) continue;- if (t < kWin) done[t] = true;- gg::mx_quant(x + (size_t) t * ldx, E, (void*) x_act(en, i));- }- }- std::function<void(int)> f[4];- for (int X = 0; X < NH; ++X)- f[X] = [&, X](int j) {- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];- const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);- const void* ap[gg::kMxCpuMaxN];- float yg[gg::kMxCpuMaxN][RB], yu[gg::kMxCpuMaxN][RB];- float *gp[gg::kMxCpuMaxN], *up[gg::kMxCpuMaxN];- for (int i = 0; i < en.n; ++i) {- ap[i] = x_act(en, i);- gp[i] = yg[i];- up[i] = yu[i];- }- // (part X's rows, from where they are: a copy's part on node X, or the file's)- gg::mx_rows(mx_part_ptr(l, en.e, 0, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, gp);- gg::mx_rows(mx_part_ptr(l, en.e, 1, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, up);- for (int i = 0; i < en.n; ++i) {- float* h = band_q ? yg[i] : c_xhd.data() + (size_t) (en.slot0 + i) * FF + r0;- for (int r = r0; r < r1; ++r) {- float gv = yg[i][r - r0], uv = yu[i][r - r0];- if (lim > 0.0f) {- gv = std::min(gv, lim);- uv = std::min(std::max(uv, -lim), lim);- }- h[r - r0] = gv / (1.0f + std::exp(-gv)) * uv;- }- if (band_q) gg::mx_quant_blocks(h, r0 / 32, r1 / 32, hq + (size_t) (en.slot0 + i) * ha, FF);- }- };- run_nodes(ne * GB, f);- if (!band_q)- for (const Ent& en : ents)- for (int i = 0; i < en.n; ++i)- gg::mx_quant(c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF, hq + (size_t) (en.slot0 + i) * ha);- for (int X = 0; X < NH; ++X)- f[X] = [&, X](int j) {- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];- const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);- const void* ap[gg::kMxCpuMaxN];- float* yp[gg::kMxCpuMaxN];- for (int i = 0; i < en.n; ++i) {- ap[i] = hq + (size_t) (en.slot0 + i) * ha;- yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + r0;- }- gg::mx_rows(mx_part_ptr(l, en.e, 2, X), dr, FF, r0 - X * Eh, r1 - X * Eh, ap, en.n, yp);- };- run_nodes(ne * DB, f);- }- // ... the low-bit ones (ds4_iq's types): as above, through ggml-cpu's dots (gg::iq_cpu_*) - each token's input- // quantized once to the weight type's activation (Q8_K) in its c_xh row's bytes, then (h whole) its h the same-#if defined(STRATA_METAL)- // The CPU lane's cold reads were 78% of its time (a madvise(WILLNEED) of a range that is not in the page cache blocks while the SSD- // reads it). Stale-router lookahead (no training): the GPU applies layer L+a's gate (+ bias, the router's own rule) to layer L's FFN- // input (a gemv and a route a layer, queued behind the shared expert) and the host reads the guessed picks with the real ones; the- // guessed CPU-lane experts whose pages are not in the page cache are read in on helper threads while this layer computes. Bytes per- // layer capped (DS4_LA_MAX experts, default 4); DS4_LOOKAHEAD layers ahead (default 2; 0 off).- static bool la_enabled() { return la_depth() > 0; }- static int la_depth() {- static const int d = std::getenv("DS4_LOOKAHEAD") ? std::atoi(std::getenv("DS4_LOOKAHEAD")) : 0; // (off by default until it is measured faster)- return std::min(d, 2);- }- struct LaPool {- std::mutex m;- std::condition_variable cv;- std::deque<std::pair<const void*, size_t>> q;- LaPool() {- const int nth = std::getenv("DS4_LA_THREADS") ? std::atoi(std::getenv("DS4_LA_THREADS")) : 3;- for (int i = 0; i < nth; ++i)- std::thread([this] {- for (;;) {- std::pair<const void*, size_t> r;- {- std::unique_lock<std::mutex> lk(m);- cv.wait(lk, [&] { return !q.empty(); });- r = q.front();- q.pop_front();- while (q.size() > 64) q.pop_front(); // (stale reads are worth nothing)- }- const uintptr_t a = (uintptr_t) r.first & ~(uintptr_t) 16383;- const size_t len = r.second + ((uintptr_t) r.first - a);- std::vector<unsigned char> vec((len + 16383) / 16384);- bool cold = true;- if (mincore((void*) a, len, (char*) vec.data()) == 0) {- cold = false;- for (unsigned char c : vec) if (!(c & 1)) { cold = true; break; }- }- if (cold) os::prefetch({r});- }- }).detach();- }- void push(const void* p, size_t n) {- { std::lock_guard<std::mutex> lk(m); q.push_back({p, n}); }- cv.notify_one();- }- };- std::mutex la_mu;- std::vector<std::vector<int>> la_guess; // per layer: the CPU-lane experts its last lookahead guessed- long la_hit = 0, la_all = 0, la_issued = 0;- void lookahead_prefetch(Layer& l, cudaEvent_t ev2, int la_n, const std::vector<Ent>& ents) {- if (l.nm != 1) return;- static LaPool* pool = nullptr;- static bool reg = false;- if (pool == nullptr) {- la_guess.assign(L.size(), {});- pool = new LaPool();- }- if (std::getenv("DS4_CPU_LOG") && !reg) {- reg = true;- static Impl* self = this;- std::atexit([] { std::fprintf(stderr, "lookahead: %ld of %ld CPU-lane experts were guessed; %ld matrices queued\n", self->la_hit, self->la_all, self->la_issued); });- }- for (const Ent& en : ents) {- ++la_all;- const auto& gs = la_guess[(size_t) l.il];- if (std::find(gs.begin(), gs.end(), en.e) != gs.end()) ++la_hit;- }- if (la_n <= 0) return;- cudaEventSynchronize(ev2);- static const int cap = std::getenv("DS4_LA_MAX") ? std::atoi(std::getenv("DS4_LA_MAX")) : 4;- for (int a = 1; a <= la_n; ++a) {- Layer& nx = L[(size_t) (l.il + a)];- std::vector<int> cpu;- for (int r = 0; r < nx.k_exp; ++r) {- const int e = h_idx_a[(a - 1) * kPicks + r];- if (e >= 0 && e < nx.n_exp && nx.slot[(size_t) e] < 0 && (int) cpu.size() < cap) cpu.push_back(e);- }- la_guess[nx.il] = cpu;- for (int e : cpu)- for (int r = 0; r < 3; ++r) { pool->push(mx_mat(nx, e, r), mx_part_bytes(nx, r)); ++la_issued; }- }- }-#endif- void cpu_experts_iq(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {- constexpr int RB = 64; // rows a job- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;- const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();- gg::IqCpu fm;- std::string ferr;- if (!gg::iq_cpu_fmt(gg::expert_type(l.lay, 0), gg::expert_type(l.lay, 2), E, FF, fm, ferr) ||- fm.act_bytes > (size_t) E * 4 || fm.h_bytes > (size_t) E * 4) {- static std::atomic<bool> said{false};- if (!said.exchange(true)) std::fprintf(stderr, "ds4: layer %d's CPU-lane experts: %s\n", l.il, ferr.c_str());- for (const Ent& en : ents) std::memset(c_yd.data() + (size_t) en.slot0 * E, 0, (size_t) en.n * E * 4);- return;- }- static double t_pre = 0, t_q1 = 0, t_r1 = 0, t_q2 = 0, t_r2 = 0; static long n_calls = 0; static bool reg = false;- if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu_experts_iq phases (ms over %ld calls): prefetch %.1f, quant1 %.1f, gate_up %.1f, quant2 %.1f, down %.1f\n", n_calls, t_pre, t_q1, t_r1, t_q2, t_r2); }); }- ++n_calls;- double tp0 = now_ms();- const float lim = g.lim;- { // (os::prefetch, as above)- std::vector<std::pair<const void*, size_t>> want;- for (const Ent& en : ents)- if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))- for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});- if (std::getenv("DS4_CPU_LOG")) { // how many of the experts' matrices are in the page cache (mincore, the first 64 pages)- static long cold_n = 0, warm_n = 0; static double cold_ms = 0, warm_ms = 0; static bool reg2 = false;- if (!reg2) { reg2 = true; std::atexit([] { std::fprintf(stderr, "prefetch: %ld matrices cold (%.1f ms), %ld warm (%.1f ms)\n", cold_n, cold_ms, warm_n, warm_ms); }); }- for (const auto& w : want) {- const uintptr_t a = (uintptr_t) w.first & ~(uintptr_t) 16383;- unsigned char vec[64];- const size_t len = std::min<size_t>(w.second, 64 * 16384);- bool cold = false;- if (mincore((void*) a, len, (char*) vec) == 0) for (size_t i = 0; i < (len + 16383) / 16384; ++i) if (!(vec[i] & 1)) cold = true;- const double t0 = now_ms();- os::prefetch({w});- (cold ? cold_ms : warm_ms) += now_ms() - t0;- (cold ? cold_n : warm_n)++;- }- } else if (want.size() > 1 && std::getenv("DS4_SERIAL_PREFETCH") == nullptr) {- // (a madvise(WILLNEED) of a cold range blocks while the SSD reads it: queue depth 1 reads 3.9 GB/s; the matrices of- // all the call's experts at once keep the SSD busy)- std::function<void(int)> pf[4];- for (int X = 0; X < NH; ++X) pf[X] = [&](int j) { os::prefetch({want[(size_t) j]}); };- run_nodes((int) want.size(), pf);- } else- os::prefetch(want);- }- double tp1 = now_ms(); t_pre += tp1 - tp0;- // the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of- // entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or- // chunk paths), then (x no longer needed) the slot's h- uint8_t* aq = (uint8_t*) c_xh.data();- auto x_act = [&](const Ent& en, int i) -> uint8_t* {- const int t = en.tok[i];- return aq + (size_t) (t < kWin ? t : kWin + en.slot0 + i) * E * 4;- };- {- bool done[kWin] = {};- for (const Ent& en : ents)- for (int i = 0; i < en.n; ++i) {- const int t = en.tok[i];- if (t < kWin && done[t]) continue;- if (t < kWin) done[t] = true;- gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));- }- }- double tq1 = now_ms(); t_q1 += tq1 - tp1;- std::function<void(int)> f[4];- for (int X = 0; X < NH; ++X)- f[X] = [&, X](int j) {- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];- const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);- const void* ap[gg::kMxCpuMaxN];- float* hp[gg::kMxCpuMaxN];- for (int i = 0; i < en.n; ++i) {- ap[i] = x_act(en, i);- hp[i] = c_xhd.data() + (size_t) (en.slot0 + i) * FF + (size_t) X * FFh; // (part X's rows of h)- }- // (part X's rows, from where they are: a copy's part on node X, or the file's - whole rows, so any- // block size of the type)- gg::iq_cpu_gate_up(fm, mx_part_ptr(l, en.e, 0, X), mx_part_ptr(l, en.e, 1, X), ap, en.n, hp,- r0 - X * FFh, r1 - X * FFh, lim);- };- run_nodes(ne * GB, f);- double tr1 = now_ms(); t_r1 += tr1 - tq1;- for (const Ent& en : ents)- for (int i = 0; i < en.n; ++i)- gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,- aq + (size_t) (kWin + en.slot0 + i) * E * 4);- double tq2 = now_ms(); t_q2 += tq2 - tr1;- for (int X = 0; X < NH; ++X)- f[X] = [&, X](int j) {- const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];- const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);- const void* hp[gg::kMxCpuMaxN];- float* yp[gg::kMxCpuMaxN];- for (int i = 0; i < en.n; ++i) {- hp[i] = aq + (size_t) (kWin + en.slot0 + i) * E * 4;- yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + (size_t) X * Eh; // (part X's rows of it)- }- gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);- };- run_nodes(ne * DB, f);- t_r2 += now_ms() - tq2;- }-- // the window's CPU-lane experts made ready (and pinned) in the store for the length of the call- struct StoreHold {- Impl* m = nullptr;- int layer = 0;- std::vector<int> es;- std::vector<char> ok;- StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {- if (!m_->store) return;- m = m_;- layer = l.il;- for (const Ent& en : ents)- if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);- m->store->ensure(layer, es.data(), (int) es.size(), ok);- }- ~StoreHold() {- if (m) m->store->release(layer, es.data(), (int) es.size(), ok);- }- };- void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,- const int* ord = nullptr) {- if (ldx == 0) ldx = (size_t) g.dim;- if (l.lay.fmt == gg::kFmtMXFP4) {- StoreHold hold(this, l, ents);- return cpu_experts_mx(l, x, ents, ldx, ord);- }- const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;- const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;- const int ne = (int) ents.size();- const float lim = g.lim;- auto part = [&](int e, int X, int r) {- const uint8_t* b = l.hh[(size_t) e][(size_t) X];- kc::Exl3Part p;- p.trellis = (const uint32_t*) (b + l.hl.trellis[r]);- p.suh = (const uint16_t*) (b + l.hl.suh[r]);- p.svh = (const uint16_t*) (b + l.hl.svh[r]);- p.k = l.hl.k[r];- p.n = l.hl.n[r];- p.K = l.lay.K[r];- return p;- };- auto xh = [&](int X, int sl, int r) { return c_xh.data() + ((size_t) (X * kSlots + sl) * 2 + r) * E; };- auto xsb = [&](int X, int sl, int r) { return c_xsb.data() + ((size_t) (X * kSlots + sl) * 2 + r) * EB; };- auto acc = [&](int X, int sl, int r, int q) {- return c_acc.data() + (((size_t) (X * kSlots + sl) * 2 + r) * GQ + q) * FFh;- };- auto xhd = [&](int sl) { return c_xhd.data() + (size_t) sl * FF; };- auto xsd = [&](int sl) { return c_xsd.data() + (size_t) sl * FB; };- auto accd = [&](int X, int sl, int q) { return c_accd.data() + ((size_t) (X * kSlots + sl) * DQ + q) * Eh; };- std::function<void(int)> f[4];- for (int X = 0; X < NH; ++X)- f[X] = [&, X](int j) {- const Ent& en = ents[(size_t) (j / (2 * GQ))];- const int r = (j / GQ) % 2, q = j % GQ;- const kc::Exl3Part p = part(en.e, X, r);- const float* xp[kc::kExl3CpuMaxN];- float* ap[kc::kExl3CpuMaxN];- for (int i = 0; i < en.n; ++i) {- const int sl = en.slot0 + i;- const float* xt = x + (size_t) en.tok[i] * ldx;- for (int b = 2 * q; b < 2 * q + 2; ++b)- xsb(X, sl, r)[b] = kc::exl3_cpu_prepare_block(p.suh + b * 128, xt + b * 128, xh(X, sl, r) + b * 128);- xp[i] = xh(X, sl, r);- ap[i] = acc(X, sl, r, q);- }- if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 16, (q + 1) * 16, ap[0]);- else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 16, (q + 1) * 16, ap);- };- run_nodes(ne * 2 * GQ, f);- for (int X = 0; X < NH; ++X)- f[X] = [&, X](int j) {- const Ent& en = ents[(size_t) (j / GB)];- const int b = j % GB;- const kc::Exl3Part pg = part(en.e, X, 0), pu = part(en.e, X, 1), pd = part(en.e, X, 2);- for (int i = 0; i < en.n; ++i) {- const int sl = en.slot0 + i;- float y[2][128], a[128], hb[128];- for (int r = 0; r < 2; ++r) {- for (int c = 0; c < 128; ++c) a[c] = 0.0f;- for (int q = 0; q < GQ; ++q) {- const float* aq = acc(X, sl, r, q) + b * 128;- for (int c = 0; c < 128; ++c) a[c] += aq[c];- }- double xs = 0.0;- for (int k = 0; k < EB; ++k) xs += xsb(X, sl, r)[k];- kc::exl3_cpu_band_out(r == 0 ? pg : pu, a, (float) xs, b * 128, y[r]);- }- for (int c = 0; c < 128; ++c) {- float gv = y[0][c], uv = y[1][c];- if (lim > 0.0f) {- gv = std::min(gv, lim);- uv = std::min(std::max(uv, -lim), lim);- }- hb[c] = gv / (1.0f + std::exp(-gv)) * uv;- }- const int gb = X * GB + b;- xsd(sl)[gb] = kc::exl3_cpu_prepare_block(pd.suh + gb * 128, hb, xhd(sl) + gb * 128);- }- };- run_nodes(ne * GB, f);- for (int X = 0; X < NH; ++X)- f[X] = [&, X](int j) {- const Ent& en = ents[(size_t) (j / DQ)];- const int q = j % DQ;- const kc::Exl3Part p = part(en.e, X, 2);- const float* xp[kc::kExl3CpuMaxN];- float* ap[kc::kExl3CpuMaxN];- for (int i = 0; i < en.n; ++i) {- xp[i] = xhd(en.slot0 + i);- ap[i] = accd(X, en.slot0 + i, q);- }- if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 8, (q + 1) * 8, ap[0]);- else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 8, (q + 1) * 8, ap);- };- run_nodes(ne * DQ, f);- for (int X = 0; X < NH; ++X)- f[X] = [&, X](int j) {- const Ent& en = ents[(size_t) (j / DB)];- const int b = j % DB;- const kc::Exl3Part pd = part(en.e, X, 2);- for (int i = 0; i < en.n; ++i) {- const int sl = en.slot0 + i;- float a[128];- for (int c = 0; c < 128; ++c) a[c] = 0.0f;- for (int q = 0; q < DQ; ++q) {- const float* aq = accd(X, sl, q) + b * 128;- for (int c = 0; c < 128; ++c) a[c] += aq[c];- }- double xs = 0.0;- for (int k = 0; k < FB; ++k) xs += xsd(sl)[k];- kc::exl3_cpu_band_out(pd, a, (float) xs, b * 128, c_yd.data() + (size_t) sl * E + (size_t) X * Eh + b * 128);- }- };- run_nodes(ne * DB, f);- }-- // ---- the expert cache. A swap moves a CPU-lane expert (`in`) into the layer's spare VRAM slot and the VRAM expert it- // replaces (`out`) into the spare host parts - both copies of weights that do not change, made while the model runs- // (the copy stream, the host's repacking) - and cache_tick switches the tables between windows, so the device's- // table and the host's view of a layer always agree within a window. In place (file_experts) `out` needs no copy:- // the CPU lane reads it from the file again.- void ram_loop() { // the RAM tier's moves in: an expert read from the file into a free slot- for (;;) {- RamSwap rs;- {- std::unique_lock<std::mutex> lk(sw_mu);- ram_cv.wait(lk, [&] { return quit.load() || !ram_queue.empty(); });- if (quit.load()) return;- rs = ram_queue.front();- ram_queue.pop_front();- }- const Layer& l = L[(size_t) rs.layer];- if (read_mx_file(l, rs.in, ram_slots[(size_t) rs.slot])) {- for (int r = 0; r < 3; ++r) // (one copy of it in RAM)- os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) rs.in * l.mx_stride[r],- mx_part_bytes(l, r), false);- } else {- rs.in = -1; // (failed: the slot goes back)- }- std::lock_guard<std::mutex> lk(sw_mu);- ram_ready.push_back(rs);- }- }-- void cache_loop(SwapBufs& B) {- for (;;) {- Swap sw;- {- std::unique_lock<std::mutex> lk(sw_mu);- sw_cv.wait(lk, [&] { return quit.load() || !sw_queue.empty(); });- if (quit.load()) return;- sw = sw_queue.front();- sw_queue.pop_front();- }- Layer& l = L[(size_t) sw.layer];- if (l.lay.fmt == gg::kFmtMXFP4) { // in: from the file; out: nothing to keep (the file has it)- swap_in_mx(l, sw, B);- std::lock_guard<std::mutex> lk(sw_mu);- sw_ready.push_back(sw);- continue;- }- parts_to_blob(l, l.hh[(size_t) sw.in], B.a);- if (l.nm == 1) {- const Dev& d = devs[(size_t) l.dev];- cudaSetDevice(d.id);- cudaStream_t cs = B.st[(size_t) l.dev];- cudaMemcpyAsync(l.vram_base + (size_t) l.spare_vram * l.blob, B.a, l.blob, cudaMemcpyHostToDevice, cs);- if (!file_experts)- cudaMemcpyAsync(B.b, l.vram_base + (size_t) l.slot[(size_t) sw.out] * l.blob, l.blob,- cudaMemcpyDeviceToHost, cs);- cudaStreamSynchronize(cs);- } else { // a group: each member's part in, each member's part out- for (int m = 0; m < l.nm; ++m) {- const Layer& x = member(l, m);- cudaSetDevice(devs[(size_t) x.dev].id);- cudaStream_t cs = B.st[(size_t) x.dev];- blob_to_half(l, B.a, m, B.h);- cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * x.hblob, B.h, x.hblob, cudaMemcpyHostToDevice, cs);- cudaStreamSynchronize(cs);- if (file_experts) continue;- cudaMemcpyAsync(B.h, x.vram_base + (size_t) l.slot[(size_t) sw.out] * x.hblob, x.hblob,- cudaMemcpyDeviceToHost, cs);- cudaStreamSynchronize(cs);- half_to_blob(l, B.h, m, B.b);- }- }- if (!file_experts)- for (int X = 0; X < NH; ++X) blob_to_part(l, B.b, X, l.spare_host[(size_t) X]);- if (check_swaps) { // DS4_CHECK_SWAPS: both moves against the model file, byte for byte- std::vector<uint8_t> reg, blob(l.blob), part(l.hl.bytes);- bool ok = true;- for (int pass = 0; pass < (file_experts ? 1 : 2); ++pass) { // (in place: no host copy of `out`)- const int e = pass == 0 ? sw.in : sw.out;- reg.resize(l.region_bytes[(size_t) e]);- pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], reg.size(), reg.data());- if (pass == 0) {- repack(l, reg.data(), blob.data());- std::vector<uint8_t> dev(l.blob);- if (l.nm == 1)- cudaMemcpy(dev.data(), l.vram_base + (size_t) l.spare_vram * l.blob, l.blob, cudaMemcpyDeviceToHost);- else- for (int m = 0; m < l.nm; ++m) { // (the members' parts, put together again)- std::vector<uint8_t> h(member(l, m).hblob);- cudaSetDevice(devs[(size_t) member(l, m).dev].id);- cudaMemcpy(h.data(), member(l, m).vram_base + (size_t) l.spare_vram * member(l, m).hblob, member(l, m).hblob,- cudaMemcpyDeviceToHost);- half_to_blob(l, h.data(), m, dev.data());- }- for (int r = 0; r < 3; ++r)- ok = ok && std::memcmp(dev.data() + l.lay.trellis[r], blob.data() + l.lay.trellis[r], l.piece_bytes[r][0]) == 0 &&- std::memcmp(dev.data() + l.lay.suh[r], blob.data() + l.lay.suh[r], l.piece_bytes[r][1]) == 0 &&- std::memcmp(dev.data() + l.lay.svh[r], blob.data() + l.lay.svh[r], l.piece_bytes[r][2]) == 0;- } else {- for (int X = 0; X < NH; ++X) {- repack_part(l, reg.data(), X, part.data());- for (int r = 0; r < 3; ++r) {- const size_t tb = (size_t) (l.hl.k[r] / 16) * (l.hl.n[r] / 16) * 32 * l.lay.K[r];- ok = ok && std::memcmp(part.data() + l.hl.trellis[r], l.spare_host[(size_t) X] + l.hl.trellis[r], tb) == 0 &&- std::memcmp(part.data() + l.hl.suh[r], l.spare_host[(size_t) X] + l.hl.suh[r], (size_t) l.hl.k[r] * 2) == 0 &&- std::memcmp(part.data() + l.hl.svh[r], l.spare_host[(size_t) X] + l.hl.svh[r], (size_t) l.hl.n[r] * 2) == 0;- }- }- }- }- std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out: %s\n", l.il, sw.in, sw.out,- ok ? "both match the model file" : "MISMATCH");- }- std::lock_guard<std::mutex> lk(sw_mu);- sw_ready.push_back(sw);- }- }-- // a GGUF's swap: expert sw.in from the file (copy_mode: its copy) into the spare VRAM slot (each member its part);- // the expert it replaces goes back to the CPU lane, which reads it from the file's mapping - or (copy_mode, sw.ram)- // from the layer's spare RAM slot, which it is copied into from VRAM here- void swap_in_mx(Layer& l, Swap& sw, SwapBufs& B) {- uint8_t *sw_a = B.a, *sw_b = B.b, *sw_h = B.h; // (this thread's)- const std::vector<cudaStream_t>& copy_st = B.st;- bool ok = copy_mode ? read_mx(l, sw.in, sw_a) : read_mx_file(l, sw.in, sw_a); // (a copy holds its slot till- // this swap's switch)- for (int m = 0; m < l.nm && ok; ++m) {- const Layer& x = member(l, m);- cudaSetDevice(devs[(size_t) x.dev].id);- cudaStream_t cs = copy_st[(size_t) x.dev];- if (l.nm > 1) blob_to_half(l, sw_a, m, sw_h);- ok = cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * unit(l), l.nm > 1 ? sw_h : sw_a, unit(l),- cudaMemcpyHostToDevice, cs) == cudaSuccess &&- cudaStreamSynchronize(cs) == cudaSuccess;- }- if (sw.ram >= 0) { // `out` from its VRAM slot (each member its part) into the spare RAM slot- bool got = ok;- for (int m = 0; m < l.nm && got; ++m) {- const Layer& x = member(l, m);- cudaSetDevice(devs[(size_t) x.dev].id);- cudaStream_t cs = copy_st[(size_t) x.dev];- got = cudaMemcpyAsync(l.nm > 1 ? sw_h : sw_b, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l),- unit(l), cudaMemcpyDeviceToHost, cs) == cudaSuccess &&- cudaStreamSynchronize(cs) == cudaSuccess;- if (got && l.nm > 1) half_to_blob(l, sw_h, m, sw_b);- }- if (got) blob_to_ram(l, sw_b, sw.ram);- else sw.ram = -1; // (back to the file's pages then)- }- if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for byte- std::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);- for (int m = 0; m < l.nm && ok; ++m) {- const Layer& x = member(l, m);- cudaSetDevice(devs[(size_t) x.dev].id);- cudaMemcpy(l.nm > 1 ? h.data() : dev.data(), x.vram_base + (size_t) l.spare_vram * unit(l), unit(l),- cudaMemcpyDeviceToHost);- if (l.nm > 1) half_to_blob(l, h.data(), m, dev.data());- }- ok = ok && read_mx_file(l, sw.in, file.data());- for (int r = 0; r < 3 && ok; ++r)- ok = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;- bool ok_out = true;- if (sw.ram >= 0) {- ok_out = read_mx_file(l, sw.out, file.data());- ram_to_blob(l, sw.ram, dev.data());- for (int r = 0; r < 3 && ok_out; ++r)- ok_out = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;- }- std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out%s: %s\n", l.il, sw.in, sw.out,- sw.ram >= 0 ? " (into RAM)" : "", ok && ok_out ? "matches the model file" : "MISMATCH");- } else if (!ok) {- std::fprintf(stderr, "ds4 cache: layer %d expert %d: the move into VRAM failed\n", l.il, sw.in);- }- }-- void cache_tick(int T) {- cache_tick_swaps(T);- store_sync();- store_watch();- }- // the store gives RAM back when the PC runs short (the owner opened something): below 4 GB free, a 0.7 GB chunk at a time- // until 5 GB are free again (checked every 16 windows); it grows again by itself once 8 GB beside its chunk are free- void store_watch() {- if (!store) return;- static int tick = 0;- if (++tick % 16 != 0) return;- const uint64_t avail = ram_avail();- if (avail == ~uint64_t{0} || avail >= (4ull << 30)) return;- const size_t chunks = 1 + (size_t) (((5ull << 30) - avail) / (700ull << 20));- const size_t n = store->shrink_chunks(chunks);- if (n > 0) std::fprintf(stderr, "ds4: RAM store: %.1f GB free: gave back %zu chunks (%zu slots left)\n", avail / 1e9, n, store->slots_allocated());- }- void cache_tick_swaps(int T) {- if (!o.expert_cache || cache_ths.empty()) return;- std::vector<Swap> ready;- std::vector<RamSwap> rready;- {- std::lock_guard<std::mutex> lk(sw_mu);- ready.swap(sw_ready);- rready.swap(ram_ready);- }- for (const RamSwap& rs : rready) { // (nothing reads the tier between windows)- --ram_inflight;- Layer& l = L[(size_t) rs.layer];- if (rs.in < 0 || l.ram[(size_t) rs.in] >= 0 || l.slot[(size_t) rs.in] >= 0) {- ram_free.push_back(rs.slot);- continue;- }- l.ram[(size_t) rs.in] = rs.slot;- if (rs.out >= 0 && l.ram[(size_t) rs.out] >= 0) {- ram_free.push_back(l.ram[(size_t) rs.out]);- l.ram[(size_t) rs.out] = -1;- }- ++st.ram_moves;- }- for (const Swap& sw : ready) {- Layer& l = L[(size_t) sw.layer];- const int old_slot = l.slot[(size_t) sw.out];- const std::array<uint8_t*, 4> old_parts = l.hh[(size_t) sw.in];- if (copy_mode) { // (`out` takes the RAM place `in` leaves: through the spare, which `in`'s slot becomes)- const int s_in = l.ram[(size_t) sw.in];- l.ram[(size_t) sw.in] = -1;- if (sw.ram >= 0) {- l.ram[(size_t) sw.out] = sw.ram;- copy_spare[(size_t) l.il] = s_in;- } else if (s_in >= 0) { // (the move into RAM failed: `out` back to the file, one copy fewer)- --ram_held;- if (copy_spare[(size_t) l.il] < 0) copy_spare[(size_t) l.il] = s_in;- }- } else if (!l.ram.empty() && l.ram[(size_t) sw.in] >= 0) { // (in VRAM now: its RAM slot for the next hottest)- ram_free.push_back(l.ram[(size_t) sw.in]);- l.ram[(size_t) sw.in] = -1;- }- l.slot[(size_t) sw.in] = l.spare_vram;- if (!l.zc.empty() && l.zc[(size_t) sw.in]) { // (VRAM's now: not the store's to compute any more)- l.zc[(size_t) sw.in] = 0;- store->unmark(l.il, sw.in);- }- if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);- l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};- l.host[(size_t) sw.in] = nullptr;- l.slot[(size_t) sw.out] = -1;- if (l.lay.fmt == gg::kFmtMXFP4) { // (the file has it)- l.host[(size_t) sw.out] = (uint8_t*) mx_mat(l, sw.out, 0);- } else if (file_experts) { // (back to its pages in the file)- l.hh[(size_t) sw.out] = {file_part(l, sw.out), nullptr, nullptr, nullptr};- l.host[(size_t) sw.out] = l.hh[(size_t) sw.out][0];- } else {- l.hh[(size_t) sw.out] = l.spare_host;- l.host[(size_t) sw.out] = l.spare_host[0];- l.spare_host = old_parts;- }- l.spare_vram = old_slot;- for (int m = 0; m < l.nm; ++m) {- Layer& x = member(l, m);- const Dev& dm = devs[(size_t) x.dev];- l.tab_pin[2 * m] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) sw.in] * unit(x));- l.tab_pin[2 * m + 1] = 0ull;- cudaSetDevice(dm.id); // (stream order: before the next window)- cudaMemcpyAsync(x.vram_tab + sw.in, &l.tab_pin[2 * m], 8, cudaMemcpyHostToDevice, dm.st);- cudaMemcpyAsync(x.vram_tab + sw.out, &l.tab_pin[2 * m + 1], 8, cudaMemcpyHostToDevice, dm.st);- }- l.swap_pending = false;- --sw_inflight;- ++st.swaps;- }- decay_tokens += T;- if (decay_tokens >= 256) { // older use counts less: x0.9 every 256 tokens- decay_tokens = 0;- for (Layer& l : L)- for (float& h : l.heat) h *= c_decay;- }- bool queued = false, rqueued = false;- for (Layer& l : L) {- if (l.cpu_hits.empty()) continue;- std::vector<int> hits;- hits.swap(l.cpu_hits);- int best = -1, vin = -1;- for (int e : hits)- if (l.slot[(size_t) e] < 0 && (best < 0 || l.heat[(size_t) e] > l.heat[(size_t) best])) best = e;- if (best >= 0 && l.spare_vram >= 0 && !l.swap_pending && sw_inflight < c_inflight) { // VRAM: the hottest in- int cold = -1;- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] >= 0 && (cold < 0 || l.heat[(size_t) e] < l.heat[(size_t) cold])) cold = e;- if (cold >= 0 && l.heat[(size_t) best] > l.heat[(size_t) cold] * c_margin + c_bonus) {- l.swap_pending = true;- ++sw_inflight;- vin = best;- Swap sw{l.il, best, cold};- if (copy_mode && l.ram[(size_t) best] >= 0) sw.ram = copy_spare[(size_t) l.il]; // (-1: none)- std::lock_guard<std::mutex> lk(sw_mu);- sw_queue.push_back(sw);- queued = true;- }- }- if (l.ram.empty() || ram_free.empty() || ram_inflight >= kMaxSwaps) continue;- int rb = -1; // RAM: the hottest read from the file, in place of the layer's coldest held (or a free slot)- for (int e : hits)- if (e != vin && l.slot[(size_t) e] < 0 && l.ram[(size_t) e] < 0 &&- (rb < 0 || l.heat[(size_t) e] > l.heat[(size_t) rb]))- rb = e;- if (rb < 0) continue;- int rc = -1;- for (int e = 0; e < l.n_exp; ++e)- if (l.ram[(size_t) e] >= 0 && (rc < 0 || l.heat[(size_t) e] < l.heat[(size_t) rc])) rc = e;- const bool room = ram_free.size() > 1; // (slots VRAM freed: filled at once)- if (!room && !(rc >= 0 && l.heat[(size_t) rb] > l.heat[(size_t) rc] * 1.25f + 1.0f)) continue;- RamSwap rs;- rs.layer = l.il;- rs.in = rb;- rs.out = room ? -1 : rc;- rs.slot = ram_free.back();- ram_free.pop_back();- ++ram_inflight;- std::lock_guard<std::mutex> lk(sw_mu);- ram_queue.push_back(rs);- rqueued = true;- }- if (queued) sw_cv.notify_all();- if (rqueued) ram_cv.notify_one();- }-- // lookahead's thread: the ranges the CPU lane asked for, read in (os::prefetch) off the lane, which computes- void prefetch_loop() {- std::vector<std::pair<const void*, size_t>> b;- for (;;) {- {- std::unique_lock<std::mutex> lk(pf_mu);- pf_cv.wait(lk, [&] { return quit.load(std::memory_order_relaxed) || !pf_q.empty(); });- if (quit.load(std::memory_order_relaxed)) return;- b.swap(pf_q);- }- os::prefetch_cold(b); // (the resident ones would only cost the page walk)- b.clear();- }- }- // an expert the CPU lane reads from the files (neither in VRAM nor in the RAM tier)- static bool file_read(const Layer& l, int e) {- return l.slot[(size_t) e] < 0 && (l.ram.empty() || l.ram[(size_t) e] < 0);- }- // the layer whose picks layer l0 guesses a layers ahead (1, 2): the next ones of its stack on its device, MXFP4- // experts read in place, some of them from the files now (none: nothing to read early - null)- const Layer* ahead_layer(const Layer& l0, int a) const {- const int j = l0.il + a, end = l0.mtp ? g.nl + g.n_mtp : g.nl;- if (j >= end || j >= (int) L.size()) return nullptr;- const Layer& n = L[(size_t) j];- if (n.dev != l0.dev || n.mtp != l0.mtp || n.lay.fmt != gg::kFmtMXFP4 || n.gate_w == nullptr || n.n_exp > 1024 ||- (int) n.slot.size() < n.n_exp)- return nullptr;- for (int e = 0; e < n.n_exp; ++e)- if (file_read(n, e)) return &n;- return nullptr;- }- // lookahead on the CPU lane: request r (layer li, seq) carries the device's guesses for the next layers - the- // guessed experts read from the files asked for now (each once a window), the guesses kept for the hit counts- std::vector<std::pair<const void*, size_t>> pf_want;- void ahead_ask(int li, const Req& r, unsigned seq, int T) {- pf_want.clear();- for (int a = 1; a <= 2; ++a) {- const int m = std::min(r.na[a - 1], a == 1 ? kAhead1 : kPicks), j = li + a;- if (m <= 0 || j >= (int) L.size()) continue;- const Layer& n = L[(size_t) j];- uint8_t* fl = ahead_fl.data() + (size_t) j * 1024;- uint8_t* rk = ahead_rank.data() + (size_t) j * 1024;- if (ahead_for[(size_t) j] != seq + (unsigned) a) {- std::memset(fl, 0, 1024);- std::memset(rk, 0, 1024);- ahead_for[(size_t) j] = seq + (unsigned) a;- }- for (int t = 0; t < T; ++t)- for (int i = 0; i < m; ++i) {- const int e = r.ahead[t][(a - 1) * kAhead1 + i];- if (e < 0 || e >= n.n_exp || e >= (int) n.slot.size()) continue;- fl[e] |= a == 2 ? 4 : (i < n.k_exp ? 3 : 2);- if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);- if ((fl[e] & 8) || !file_read(n, e)) continue;- fl[e] |= 8;- if (store) store->prefetch(j, e);- else- for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});- ++st.ahead_pf;- }- }- if (pf_want.empty()) return;- {- std::lock_guard<std::mutex> lk(pf_mu);- pf_q.insert(pf_q.end(), pf_want.begin(), pf_want.end());- }- pf_cv.notify_one();- }-- // the decode path's host side: each published routing in turn - its CPU lane experts (if any) into the answer's- // part, then the answer's seq. Spins while requests come (a token's 40 layers), naps after 2 ms without one.- void serve_lane() {- unsigned next = 1;- auto last = std::chrono::steady_clock::now();- std::vector<Ent> ents;- while (!quit.load(std::memory_order_relaxed)) {- Req& r = reqs[next % kRing];- if (*(volatile unsigned*) &r.seq != next) {- cpu_relax();- if (std::chrono::steady_clock::now() - last > std::chrono::milliseconds(2))- std::this_thread::sleep_for(std::chrono::microseconds(20));- continue;- }- std::atomic_thread_fence(std::memory_order_acquire);- const int li = layer_of[next % kRing].load(std::memory_order_acquire);- Resp& rp = resps[next % kRing];- const int T = std::max(1, std::min(kWin, r.T));- for (int t = 0; t < T; ++t) rp.cpu[t] = 0;- if (li >= 0 && li < (int) L.size()) {- Layer& l = L[(size_t) li];- uint32_t* pr = prof.data() + (size_t) li * g.n_exp;- ents.clear();- if (store && li > 0 && li < g.nl) store->cancel_stale(li - 1); // (guesses for layers the token has passed: dropped unread)- // lookahead: the guesses made for this layer (when they are for this request), counted on decode- const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next- ? ahead_fl.data() + (size_t) li * 1024 : nullptr;- // the window's CPU-lane picks by expert (up to 8 tokens an entry), their weights by entry row- float wv[kWin * kPicks];- int tok_of[kWin * kPicks];- int nslots = 0;- std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only- if (trace_f != nullptr && !l.mtp) { // DS4_TRACE=file: one record a route request: u8 layer, u8 T, T x k int16 expert ids- // DS4_TRACE_AHEAD=1: T has bit 7 set and the guesses follow: u8 na1, u8 na2, T x (na1 + na2) int16 (the- // next layer's likely picks by rank, then the one after's), as the lookahead made them for THIS layer's- // predecessors (the request carries the guesses for the layers after it)- static const bool with_ahead = std::getenv("DS4_TRACE_AHEAD") != nullptr;- uint8_t hd[2] = {(uint8_t) li, (uint8_t) (T | (with_ahead ? 0x80 : 0))};- std::fwrite(hd, 1, 2, trace_f);- for (int t = 0; t < T; ++t)- for (int i = 0; i < l.k_exp; ++i) {- const int16_t v = (int16_t) r.ids[t][i];- std::fwrite(&v, 2, 1, trace_f);- }- if (with_ahead) {- const uint8_t na[2] = {(uint8_t) std::max(0, std::min(r.na[0], kAhead1)), (uint8_t) std::max(0, std::min(r.na[1], kPicks))};- std::fwrite(na, 1, 2, trace_f);- for (int t = 0; t < T; ++t) {- for (int i = 0; i < na[0]; ++i) {- const int16_t v = (int16_t) r.ahead[t][i];- std::fwrite(&v, 2, 1, trace_f);- }- for (int i = 0; i < na[1]; ++i) {- const int16_t v = (int16_t) r.ahead[t][kAhead1 + i];- std::fwrite(&v, 2, 1, trace_f);- }- }- }- }- const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;- for (int t = 0; t < T; ++t) {- float wsum = 0.0f; // (DISK_SKIP_W: the token's route weights)- if (dsk)- for (int i = 0; i < l.k_exp; ++i)- if (r.ids[t][i] >= 0 && r.ids[t][i] < l.n_exp) wsum += r.w[t][i];- for (int i = 0; i < l.k_exp; ++i) {- const int e = r.ids[t][i];- if (e < 0 || e >= l.n_exp) continue;- ++pr[e];- l.heat[(size_t) e] += 1.0f;- if (fl != nullptr) {- ++st.ahead_picks;- st.ahead_hit_k += fl[e] & 1;- st.ahead_hit += (fl[e] & 6) != 0;- st.ahead_hit2 += (fl[e] >> 2) & 1;- }- if (l.slot[(size_t) e] < 0) {- l.cpu_hits.push_back(e);- if (T == 1 && !l.zc.empty() && l.zc[(size_t) e]) { // the device computed it from the store's slot- store->touch(li, e);- ++st.store_zc;- continue;- }- if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {- ++st.skipped;- continue;- }- if (!l.ram.empty() && l.ram[(size_t) e] >= 0) ++st.ram_experts;- else if (fl != nullptr) {- ++st.ahead_cold;- st.ahead_cold_hit += (fl[e] >> 3) & 1;- }- picks.push_back({e, t * kPicks + i});- }- }- }- if (fl != nullptr) { // the guesses by rank: how many were picked- const uint8_t* rk = ahead_rank.data() + (size_t) li * 1024;- for (int e = 0; e < l.n_exp && e < 1024; ++e) {- if (rk[e] == 0 || rk[e] > 16) continue;- bool hit = false;- for (int i = 0; i < l.k_exp; ++i) hit = hit || r.ids[0][i] == e;- ++st.ahead_rank_n[rk[e] - 1];- st.ahead_rank_hit[rk[e] - 1] += hit;- }- }- if (ahead_mode > 0 && !ahead_for.empty()) ahead_ask(li, r, next, T);- std::sort(picks.begin(), picks.end());- for (size_t a = 0; a < picks.size();) {- Ent en;- en.e = picks[a].first;- en.slot0 = nslots;- en.n = 0;- while (a < picks.size() && picks[a].first == en.e && en.n < kc::kExl3CpuMaxN) {- const int t = picks[a].second / kPicks, i = picks[a].second % kPicks;- en.tok[en.n++] = t;- tok_of[nslots] = t;- wv[nslots] = r.w[t][i];- ++nslots;- ++a;- }- ents.push_back(en);- }- // lookahead: the entries asked for early (or in the RAM tier) computed first, so the ones asked for- // only now get the others' time to come in from the SSD (their slots, so the sums, stay the same)- int ord[kWin * kPicks];- const int* po = nullptr;- if (ahead_order && !ahead_for.empty() && ahead_for[(size_t) li] == next && ents.size() > 1 &&- ents.size() <= (size_t) (kWin * kPicks)) {- const uint8_t* fa = ahead_fl.data() + (size_t) li * 1024;- int n = 0;- for (int pass = 0; pass < 2; ++pass)- for (size_t q = 0; q < ents.size(); ++q) {- const int e = ents[q].e;- const bool early = !file_read(l, e) || (fa[e] & 8) != 0;- if (early == (pass == 0)) ord[n++] = (int) q;- }- po = ord;- }- if (!ents.empty()) {- const double tc = now_ms();- cpu_experts(l, &r.x[0][0], ents, (size_t) kMaxDim, po);- const int D = g.dim;- for (int t = 0; t < T; ++t) {- bool any = false;- for (int sl = 0; sl < nslots; ++sl) any = any || tok_of[sl] == t;- if (!any) continue;- float* o = rp.part[t];- for (int e = 0; e < D; ++e) o[e] = 0.0f;- for (int sl = 0; sl < nslots; ++sl) {- if (tok_of[sl] != t) continue;- const float* y = c_yd.data() + (size_t) sl * D;- for (int e = 0; e < D; ++e) o[e] += wv[sl] * y[e];- }- rp.cpu[t] = 1;- }- st.cpu_ms += now_ms() - tc;- st.cpu_experts += nslots;- }- }- std::atomic_thread_fence(std::memory_order_release);- *(volatile unsigned*) &rp.seq = next;- ++next;- last = std::chrono::steady_clock::now();- }- }-- // a GGUF's matrices (BF16 / MXFP4) to its own kernels, EXL3 ones to x3's- static bool mv_any(const x3::MvJob* j, int n, cudaStream_t s) {- bool g = false;- for (int i = 0; i < n; ++i) g = g || j[i].m->fmt != 0;- return g ? gg::mv(j, n, s) : x3::mv(j, n, s);- }- bool mv(Dev& d, std::initializer_list<x3::MvJob> jobs, std::string& err) {- x3::MvJob j[x3::kMaxMvJobs];- int n = 0;- for (const x3::MvJob& x : jobs) j[n++] = x;- if (!mv_any(j, n, d.st)) {- err = "exl3 mv failed";- return false;- }- return true;- }- bool mv_n(Dev& d, x3::MvJob* j, int n, std::string& err) {- for (int a = 0; a < n; a += x3::kMaxMvJobs)- if (!mv_any(j + a, std::min(x3::kMaxMvJobs, n - a), d.st)) {- err = "exl3 mv failed";- return false;- }- return true;- }- // y[c] = x[c] W for the T activations x + c * x_cs (floats), into y + c * y_cs- static x3::MvJob job(const x3::Mat& m, const float* x, float* y, int T = 1, size_t x_cs = 0, size_t y_cs = 0,- float alpha = 1.0f) {- x3::MvJob j;- j.m = &m;- j.xf = x;- j.y = y;- j.alpha = alpha;- j.ncol = T;- j.xf_cs = x_cs;- j.y_cs = y_cs;- return j;- }-- // ---- the DSpark drafter (DeepSeek's DSparkBlock / forward_spec). Its blocks' rings hold the trunk's main KV:- // seed() turns a window's taps (the streams' mean entering layers 37..39) into it, through main_proj / main_norm- // and each block's wkv / kv_norm / rope / FP8, for the positions the trunk kept. draft() runs the block [y, noise x- // (block - 1)] at positions pos .. through the blocks (each query attending to the ring and to the whole block),- // the trunk's head, the markov bias token by token (greedy) and the confidence head.- bool seed_drafter(int64_t p0, int n, int row0, std::string& err) {- if (g.n_mtp == 0 || n <= 0) return true;- if (n > kWin * kPicks) {- err = "seeding the drafter with " + std::to_string(n) + " rows";- return false;- }- Dev& d = devs[(size_t) d_dev];- cudaSetDevice(d.id);- const int nt = (int) g.d_targets.size(), D = g.dim, HD = g.hd;- if (dump_d != nullptr) { // DS4_DSPARK_DUMP: 'S', the first position, the rows, then the taps- for (Dev& e : devs) {- cudaSetDevice(e.id);- CK(cudaStreamSynchronize(e.st));- }- cudaSetDevice(d.id);- const int64_t hdr[4] = {'S', p0, n, (int64_t) nt * D};- std::fwrite(hdr, 8, 4, dump_d);- std::fwrite(h_taps + (size_t) row0 * nt * D, 4, (size_t) n * nt * D, dump_d);- std::fflush(dump_d);- }- for (int a = 0; a < n; a += kWin) { // (a chunk's last 128 rows: 16 a GEMV)- const int na = std::min(kWin, n - a);- CK(cudaMemcpyAsync(d.xe, h_taps + (size_t) (row0 + a) * nt * D, (size_t) na * nt * D * 4,- cudaMemcpyHostToDevice, d.st));- if (!mv(d, {job(d_main_proj, d.xe, d.ed + (size_t) a * D, na, (size_t) nt * D, D)}, err)) return false;- }- CK(cudaEventRecord(ev_seed, d.st));- Finish fm;- fm.w = d_main_norm;- fm.eps = g.eps;- finish(d.ed, n, D, D, fm, d.st);- for (int i = 0; i < g.n_mtp; ++i) {- Layer& l = L[(size_t) (g.nl + i)];- if (l.dev != d_dev) {- err = "the DSpark blocks are split across devices";- return false;- }- for (int a = 0; a < n; a += kWin) {- const int na = std::min(kWin, n - a);- if (!mv(d, {job(l.wkv, d.ed + (size_t) a * D, d.kvr, na, D, HD)}, err)) return false;- Finish fk;- fk.w = l.kv_norm;- fk.eps = g.eps;- fk.rd = g.rd;- fk.pos = (int) (p0 + a);- fk.freqs = d.freqs_win;- fk.quant = 1;- fk.dst = d.ext + (size_t) g.win * HD;- fk.dst_stride = HD;- finish(d.kvr, na, HD, HD, fk, d.st);- ext_to_ring(d.ext, l.win, p0 + a, g.win, na, HD, d.st);- }- }- return true;- }-- bool draft(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err, float temperature, float top_p,- int top_k, float min_p, uint64_t seed, uint64_t counter) {- const double t0 = now_ms();- drafts_q = temperature > 0.0f && sample_drafts;- dq_temp = temperature;- dq_top_p = top_p;- dq_top_k = top_k;- dq_min_p = min_p;- dq_seed = seed;- dq_counter = counter;- const bool ok = draft_(y, out, conf, err);- if (!ok) drafts_q = false;- if (max_drafts > 0 && (int) out.size() > max_drafts) {- out.resize((size_t) max_drafts);- conf.resize((size_t) max_drafts);- }- st.draft_ms += now_ms() - t0;- ++st.drafts_rounds;- return ok;- }- bool draft_(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err) {- out.clear();- conf.clear();- const int T = g.d_block, D = g.dim, H = g.hc, V = g.vocab, R = g.d_rank;- const int64_t p0 = pos;- if (g.n_mtp == 0 || T < 1 || T > kWin || p0 + T > o.max_context) return true;- int tk[kWin];- tk[0] = y;- for (int t = 1; t < T; ++t) tk[t] = g.d_noise;- Dev& d = devs[(size_t) d_dev];- cudaSetDevice(d.id);- for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));- embed(d.emb_rows, T, d.h, D, H, d.st);- if (std::getenv("DS4_DBG_EMB")) {- ds4_metal_flush();- float *hp = d.h;- float hh[4] = {0,0,0,0};- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);- }- if (std::getenv("DS4_DBG_EMB")) {- ds4_metal_flush();- float hh[4] = {0, 0, 0, 0};- cudaMemcpy(hh, d.h, sizeof(hh), cudaMemcpyDeviceToHost);- std::fprintf(stderr, "DBG after embed: h[0..3] = %g %g %g %g\n", hh[0], hh[1], hh[2], hh[3]);- }- init_pre(d.pre, T, d.st);- WinPlan dp; // (window attention alone: the mailbox seqs)- dp.async = !sync_decode && T <= async_max_t;- for (int i = 0; i < g.n_mtp && dp.async; ++i) {- const int il = g.nl + i;- dp.seq[il] = ++seq_issued;- layer_of[dp.seq[il] % kRing].store(il, std::memory_order_release);- }- const size_t hs = (size_t) T * H * D, rs = (size_t) g.win * g.hd;- if (dump_d != nullptr && h_dump == nullptr &&- (!halloc(h_dump, (size_t) g.n_mtp * kWin * H * D, err) || !halloc(h_dump_ring, (size_t) g.n_mtp * rs, err)))- return false;- for (int i = 0; i < g.n_mtp; ++i) {- Layer& l = L[(size_t) (g.nl + i)];- if (dump_d != nullptr) { // (stream order: the ring the block reads - as BF16 rows - the stream it leaves)- ring_to_ext(l.win, d.ext, g.win, g.win, g.hd, d.st);- CK(cudaMemcpyAsync(h_dump_ring + (size_t) i * rs, d.ext, rs * 2, cudaMemcpyDeviceToHost, d.st));- }- if (!layer(l, 0, p0, T, dp, err)) return false;- if (dump_d != nullptr) {- cudaSetDevice(d.id);- CK(cudaMemcpyAsync(h_dump + (size_t) i * hs, d.h, hs * 4, cudaMemcpyDeviceToHost, d.st));- }- }- cudaSetDevice(d.id);- hc_pre_norm(d.h, d.pre, nullptr, D, H, T, g.eps, d.x, d.st); // (the confidence head reads it before the norm)- Dev& hd = devs[(size_t) head_dev];- // (through the host to the head's device, and to the head group's other members - the drafter's device can be- // the head group's lead itself, a group of every GPU)- if (d_dev != head_dev || groups[(size_t) hd.grp].mem.size() > 1) {- CK(cudaMemcpyAsync(d.bounce, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, d.st));- CK(cudaEventRecord(d.ev_out, d.st));- }- if (d_dev != head_dev) {- cudaSetDevice(hd.id);- CK(cudaStreamWaitEvent(hd.st, d.ev_out, 0));- CK(cudaMemcpyAsync(hd.x, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, hd.st));- }- // the drafter's final norm and the trunk's head, on every member of the head group (the others' shares first)- const std::vector<int>& hg = groups[(size_t) hd.grp].mem;- const bool mk = mk_split && !drafts_q; // (sampled drafts: the lead's whole rows, gathered)- for (int m = (int) hg.size() - 1; m >= 0; --m) {- Dev& e = devs[(size_t) hg[(size_t) m]];- cudaSetDevice(e.id);- if (m > 0) {- CK(cudaStreamWaitEvent(e.st, d.ev_out, 0));- CK(cudaMemcpyAsync(e.ao, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, e.st));- } else- CK(cudaMemcpyAsync(e.ao, e.x, (size_t) T * D * 4, cudaMemcpyDeviceToDevice, e.st));- Finish fn;- fn.w = e.d_norm;- fn.eps = g.eps;- finish(e.ao, T, D, D, fn, e.st);- if (!head_share(e, m, e.ao, T, err, !mk)) return false;- }- h_idx[0] = y;- if (mk) {- // the markov steps over the group: each member its rows of the bias and its share's argmax, every member- // then the global one (the sampler's tie rule) and its E row - the same drafts as the lead's whole rows,- // each step ~1/nm of their 66 MB and no gather of the shares- const int nm = (int) hg.size();- const unsigned seq = ++mk_seq;- for (int m = 0; m < nm; ++m) {- Dev& e = devs[(size_t) hg[(size_t) m]];- cudaSetDevice(e.id);- CK(cudaMemcpyAsync(e.mk_ints, h_idx, 4, cudaMemcpyHostToDevice, e.st));- CK(cudaMemcpyAsync(e.mk_erow, h_mk_embed + (size_t) y * R, (size_t) R * 2, cudaMemcpyHostToDevice, e.st));- }- for (int i = 0; i < T; ++i)- for (int m = 0; m < nm; ++m) {- Dev& e = devs[(size_t) hg[(size_t) m]];- cudaSetDevice(e.id);- float* row = m == 0 ? hd.logits + (size_t) i * V : e.logits_part + (size_t) i * e.mk_vn;- markov_share(row, e.mk_share, e.mk_erow, e.mk_vn, R, e.st);- argmax_share(row, e.mk_vn, e.mk_v0, mk_slots + ((size_t) i * nm + m) * 4, seq, e.st);- markov_pick(mk_slots + (size_t) i * nm * 4, nm, seq, e.mk_ints + i + 1, h_mk_embed, e.mk_erow, R,- e.st);- }- cudaSetDevice(hd.id);- } else {- CK(cudaMemcpyAsync(d_ints, h_idx, 4, cudaMemcpyHostToDevice, hd.st));- strata::dsv41k::SamplerParams sp;- sp.greedy = true;- sp.temperature = 0.0f;- if (drafts_q) { // the request's own chain: draft i drawn as the model's token i would be (verify's counter + i)- sp.greedy = false;- sp.temperature = dq_temp * (q_temp_scale > 0.0f ? q_temp_scale : 1.0f);- sp.top_p = dq_top_p;- sp.top_k = dq_top_k;- sp.min_p = dq_min_p;- sp.seed = dq_seed;- }- for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft- markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);- if (drafts_q) {- sp.counter = dq_counter + (uint64_t) i;- strata::dsv41k::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);- } else- strata::dsv41k::sample_tokens(hd.logits + (size_t) i * V, 1, V, nullptr, 0, sp, d_ints + i + 1, hd.st);- }- }- confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);- int tok[kWin + 1];- float cf[kWin];-#if defined(STRATA_METAL)- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)-#endif- CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));- CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));- CK(cudaStreamSynchronize(hd.st));- for (int i = 0; i <= T; ++i) tok[i] = h_idx[i];- for (int i = 0; i < T; ++i) cf[i] = h_w[i];- if (dump_d != nullptr) { // 'D', the block's position, y, T, then the drafts, their confidences, the hidden- std::vector<float> xs((size_t) T * D); // rows before the final norm; then each block's ring and stream- CK(cudaMemcpy(xs.data(), hd.x, xs.size() * 4, cudaMemcpyDeviceToHost));- cudaSetDevice(d.id);- CK(cudaStreamSynchronize(d.st));- const int64_t hdr[4] = {'D', p0, y, T};- std::fwrite(hdr, 8, 4, dump_d);- std::fwrite(tok + 1, 4, (size_t) T, dump_d);- std::fwrite(cf, 4, (size_t) T, dump_d);- std::fwrite(xs.data(), 4, xs.size(), dump_d);- for (int i = 0; i < g.n_mtp; ++i) {- std::fwrite(h_dump_ring + (size_t) i * rs, 2, rs, dump_d);- std::fwrite(h_dump + (size_t) i * hs, 4, hs, dump_d);- }- std::fflush(dump_d);- cudaSetDevice(hd.id);- }- for (int i = 0; i < T; ++i) {- out.push_back(tok[i + 1]);- conf.push_back(cf[i]);- }- return true;- }-- int sample_row(int row, float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter,- const float* bias) {- Dev& d = devs[(size_t) head_dev];- cudaSetDevice(d.id);- strata::dsv41k::SamplerParams sp;- sp.temperature = temperature;- sp.greedy = temperature <= 0.0f;- sp.top_p = top_p;- sp.top_k = top_k;- sp.min_p = min_p;- sp.seed = seed;- sp.counter = counter;- sp.logit_bias = bias;- strata::dsv41k::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);- int t = -1;-#if defined(STRATA_METAL)- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)-#endif- cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);- cudaStreamSynchronize(d.st);- return t;- }-- // [y, drafts] at positions pos ..: the trunk's verdict, its next token, and the state back to what it kept.- // Greedy: the drafts the trunk's argmax agrees with. Sampled: the speculative-sampling rule over the request's own- // chain (top_k, top_p, min_p, temperature): draft d stands with probability min(1, p(d) / q(d)), q the distribution- // it was drawn from (the drafter's argmax: a point at d, so p(d)), a rejected one is replaced by a draw from the- // residual max(0, p - q) - so each kept token is distributed as the trunk's own draw.- bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,- uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err) {- const int n = (int) std::min<size_t>(drafts.size(), (size_t) kWin - 1);- const int T = n + 1, V = g.vocab;- const int64_t p0 = pos;- int tk[kWin];- tk[0] = y;- for (int i = 0; i < n; ++i) tk[i + 1] = drafts[(size_t) i];- spec_win = true;- const double tv = now_ms();- const bool ok = forward(tk, T, T, err, 0);- st.verify_ms += now_ms() - tv;- spec_win = false;- if (!ok) return false;- Dev& hd = devs[(size_t) head_dev];- cudaSetDevice(hd.id);- int m = 0;- if (temperature <= 0.0f) {- strata::dsv41k::SamplerParams sp;- sp.greedy = true;- sp.temperature = 0.0f;- strata::dsv41k::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)-#if defined(STRATA_METAL)- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)-#endif- CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));- CK(cudaStreamSynchronize(hd.st));- while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;- next = h_idx[m];- } else {- // row i keeps draft i with probability min(1, p(d) / q(d)), else draws from the residual max(0, p - q)- // (never d itself): each row a token distributed as the model's own draw at that position, p through the- // request's chain - the top_k / top_p / min_p cut included (the drafter's argmax can lie outside it: p(d) 0,- // never kept); past the first rejection the rows do not count- const bool own = drafts_q && dq_temp == temperature && dq_top_p == top_p && dq_top_k == top_k &&- dq_min_p == min_p && dq_seed == seed && dq_counter == counter;- if (!own) { // the drafter's argmax: q a point at each draft- for (int i = 0; i < n; ++i) {- h_specq[(size_t) i].n = 1;- h_specq[(size_t) i].ids[0] = drafts[(size_t) i];- h_specq[(size_t) i].p[0] = 1.0;- }- CK(cudaMemcpyAsync(d_specq, h_specq.data(), (size_t) n * sizeof(strata::dsv41k::SpecList),- cudaMemcpyHostToDevice, hd.st));- }- for (int i = 0; i < n; ++i) h_idx[i] = drafts[(size_t) i];- CK(cudaMemcpyAsync(d_ints + 32, h_idx, (size_t) n * 4, cudaMemcpyHostToDevice, hd.st));- strata::dsv41k::SamplerParams sp;- sp.greedy = false;- sp.temperature = temperature;- sp.top_p = top_p;- sp.top_k = top_k;- sp.min_p = min_p;- sp.seed = seed;- sp.counter = counter;- strata::dsv41k::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);-#if defined(STRATA_METAL)- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)-#endif- CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));- CK(cudaStreamSynchronize(hd.st));- while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;- next = m < n ? h_idx[m] : sample_row(n, temperature, top_p, top_k, min_p, seed, counter + (uint64_t) n, nullptr);- }- drafts_q = false;- rollback(p0, T, m + 1);- if (!seed_drafter(p0, m + 1, 0, err)) return false;- accepted = m;- st.drafts_offered += n;- st.drafts_accepted += m;- for (int i = 0; i < n && i <= m && i < 8; ++i) {- ++st.draft_reached[i];- if (i < m) ++st.draft_kept[i];- }- return true;- }-- // a verified window keeps its first `keep` positions: the ring rows and the compressors' state of the rest undone- // (the compressed KV and the index keys past them are written again when those positions come)- void rollback(int64_t p0, int T, int keep) {- if (keep >= T) return;- // the ring rows: one launch a device for all its layers (a launch a layer and member was ~300 on 8-way tensor- // parallel, a millisecond of the host's before the next draft)- std::vector<RingBatch> rb(devs.size());- for (int il = 0; il < g.nl; ++il)- for (int m = 0; m < L[(size_t) il].nm; ++m) {- Layer& l = member(L[(size_t) il], m);- RingBatch& b = rb[(size_t) l.dev];- if (b.n == kMaxRingBatch) { // (more layers on a device than a batch takes: this one alone)- cudaSetDevice(devs[(size_t) l.dev].id);- ring_restore(l.win, l.ring_bak, p0, g.win, keep, T, g.hd, devs[(size_t) l.dev].st);- continue;- }- b.ring[b.n] = l.win;- b.bak[b.n++] = l.ring_bak;- }- for (size_t di = 0; di < devs.size(); ++di)- if (rb[di].n > 0) {- cudaSetDevice(devs[di].id);- ring_restore_batch(rb[di], p0, g.win, keep, T, g.hd, devs[di].st);- }- for (int il = 0; il < g.nl; ++il)- for (int m = 0; m < L[(size_t) il].nm; ++m) {- Layer& l = member(L[(size_t) il], m);- Dev& d = devs[(size_t) l.dev];- if (l.c_snap != nullptr) {- cudaSetDevice(d.id);- const size_t ns = (size_t) l.ratio * g.hd * 4;- cudaMemcpyAsync(l.c_kv, l.c_snap, ns, cudaMemcpyDeviceToDevice, d.st);- cudaMemcpyAsync(l.c_score, l.c_snap + (size_t) l.ratio * g.hd, ns, cudaMemcpyDeviceToDevice, d.st);- compress_win(l.c_rows, l.c_rows + (size_t) kWin * g.hd, l.c_kv, l.c_score, l.ratio, (int) p0, keep,- g.hd, l.c_norm, g.eps, d.latent, d.st);- }- }- pos = p0 + keep;- toks.resize((size_t) pos);- ctoks.resize((size_t) pos);- }-- // ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows- bool step(int token, bool want_logits, std::string& err) { return forward(&token, 1, want_logits ? 1 : 0, err); }--- // layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the- // projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in- // place, the CPU lane's streamed in batches)- bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err) {- Layer& l = member(L0, mi);- Dev& d = devs[(size_t) l.dev];- Chunk& c = *chunks[(size_t) l.dev];- cudaSetDevice(d.id);- cudaStream_t s = d.st;- prof_mark(d, 0);- const int nm = L0.nm;- const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;- const ArGroup& ab = groups[(size_t) L0.grp].arb;- const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;- // Engram, in slabs of 256 tokens- if (l.eg >= 0) {- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;- const size_t per = (size_t) cols * (ehd + ehd / 32);- const int slot = wp.eg_slot >= 0 ? wp.eg_slot : wp.k % 2; // (read ahead by the run: two in flight)- uint8_t* host = c.eg_host[slot] + (size_t) l.eg * c.C * per;- CK(cudaMemcpyAsync(c.eg_rows, host, (size_t) T * per, cudaMemcpyHostToDevice, s));- CK(cudaEventRecord(c.eg_ev[slot], s));- for (int s0 = 0; s0 < T; s0 += 256) {- const int ns = std::min(256, T - s0);- engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_fmt[(size_t) l.eg]);- x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);- if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;- engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);- }- }- // attention- if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;- dump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);- hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);- hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);- dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);- dump_sub(d, "a_pre", l.il, p0, T, c.a_pre, (size_t) T * 8 * 4);- dump_sub(d, "a_post", l.il, p0, T, c.a_post, (size_t) T * 8 * 4);- dump_sub(d, "a_comb", l.il, p0, T, c.a_comb, (size_t) T * 64 * 4);- dump_sub(d, "attn_x", l.il, p0, T, c.x, (size_t) T * D * 4);- x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);- if (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||- !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err))- return false;- if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;- if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;- dump_sub(d, "qr_raw", l.il, p0, T, c.qr, (size_t) T * g.q_lora * 4);- dump_sub(d, "kvr_raw", l.il, p0, T, c.kvr, (size_t) T * HD * 4);- Finish fq;- fq.w = l.q_norm;- fq.eps = g.eps;- finish(c.qr, T, g.q_lora, g.q_lora, fq, s);- x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);- // the window rows: the ring's (positions p0 - win ..) then the chunk's- ring_to_ext(l.win, c.ext, p0, g.win, HD, s);- Finish fk;- fk.w = l.kv_norm;- fk.eps = g.eps;- fk.rd = g.rd;- fk.pos = (int) p0;- fk.freqs = freqs;- fk.quant = 1;- fk.dst = c.ext + (size_t) g.win * HD;- fk.dst_stride = HD;- finish(c.kvr, T, HD, HD, fk, s);- // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it- const bool snap = wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;- if (snap) {- CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));- if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);- }- ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);- const uint8_t* ckv = nullptr;- const long long ss = (long long) o.max_context + 64;- if (l.ratio > 0) {- Src& sr = srcs[(size_t) l.src];- const int R = l.ratio;- const int64_t g_first = p0 / R, g_end = (p0 + T) / R;- const int ng = (int) std::max<int64_t>(0, g_end - g_first);- const float* latent = c.latent;- if (l.kv_src) {- if (R == 1) {- Finish fc;- fc.w = l.c_norm;- fc.eps = g.eps;- finish(c.ckr, T, HD, HD, fc, s);- latent = c.ckr;- } else {- if (snap && L0.snap_kv != nullptr) { // (each slot: the latest row before snap_t that holds it)- const int64_t ps = p0 + wp.snap_t;- for (int sl = 0; sl < R; ++sl) {- const int64_t q = ps - 1 - (((ps - 1 - sl) % R) + R) % R;- const float* kv = q >= p0 ? c.ckr + (size_t) (q - p0) * HD : l.c_kv + (size_t) sl * HD;- const float* sc = q >= p0 ? c.cgr + (size_t) (q - p0) * HD : l.c_score + (size_t) sl * HD;- CK(cudaMemcpyAsync(L0.snap_kv + (size_t) sl * HD, kv, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));- CK(cudaMemcpyAsync(L0.snap_sc + (size_t) sl * HD, sc, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));- }- }- compress_win(c.ckr, c.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, c.latent, s);- }- }- const int clen_max = (int) ((p0 + T) / R);- if (l.ix_src) {- Src& ks = srcs[(size_t) l.ik_src];- if (l.owns_k && ng > 0) {- x3::rows_f16(latent, HD, nullptr, ng, HD, c.lat16, s);- if (!gemm(d, c, l.i_wk, c.lat16, HD, ng, c.ikr, g.ix_hd, 0.0f, err)) return false;- Finish fi;- fi.w = l.ik_norm;- fi.eps = g.eps;- fi.rd = g.rd;- fi.pos = (int) (g_first * R);- fi.pos_step = R;- fi.freqs = freqs;- fi.quant = 3;- fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();- fi.pdst_stride = (long long) ik_rb();- finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);- }- if (clen_max > 0) {- if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,- 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))- return false;- for (int a0 = 0; a0 < T; a0 += c.AI) {- const int na = std::min(c.AI, T - a0);- const int64_t pa = p0 + a0;- if (!gemm(d, c, l.i_wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.iq,- g.ix_heads * g.ix_hd, 0.0f, err))- return false;- Finish fi;- fi.rd = g.rd;- fi.pos = (int) pa;- fi.vpp = g.ix_heads;- fi.freqs = freqs;- fi.quant = 3;- finish(c.iq, na * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);- Rows lim;- lim.gen = 1;- lim.p0 = pa;- lim.R = R;- lim.kcap = g.ix_topk;- lim.ccap = g.cand_topk;- lim.blk = std::max(1, g.cand_blk);- const int nlast = (int) ((pa + na) / R); // the slab's widest query- int* tk = c.topk + (size_t) a0 * topk_stride();- int* cd = c.cand + (size_t) a0 * cand_stride();- const float* iw = c.iw + (size_t) a0 * 64;- if (nlast <= 0) continue;- if (l.uses_cand && (nlast + g.cand_blk - 1) / g.cand_blk > g.cand_topk) {- Rows cl = lim;- cl.gen = 2;- index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, g.cand_topk * g.cand_blk,- cd, (int) cand_stride(), g.cand_blk, cl, c.scores, ss, s);- Rows tr = lim;- tr.gen = 3;- topk(c.scores, ss, na, tr, cd, (int) cand_stride(), g.cand_blk, tk, (int) topk_stride(), s);- } else {- index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, nlast, nullptr, 0, 1,- lim, c.scores, ss, s);- if (l.cand_src) {- Rows cr = lim;- cr.kcap = g.cand_topk;- cand_blocks(c.scores, ss, na, cr, g.cand_blk, c.bm, (long long) bm_stride(), cd,- (int) cand_stride(), s);- }- topk(c.scores, ss, na, lim, nullptr, 0, 1, tk, (int) topk_stride(), s);- }- }- }- }- if (l.kv_src && ng > 0) {- Finish fc;- fc.rd = g.rd;- fc.pos = (int) (g_first * R);- fc.pos_step = R;- fc.freqs = freqs;- fc.quant = 2;- fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();- fc.pdst_stride = (long long) ckv_rb();- finish(const_cast<float*>(latent), ng, HD, HD, fc, s);- if (mi == 0 && sr.remote) { // the new rows for the other groups' readers (slot k % 2, once- const int slot = wp.k % 2; // they took the chunk before last's)- if (wp.k >= 2)- for (int r : sr.readers) wait_flags(sr.x_ack + ((size_t) slot * devs.size() + r) * 32, 1, 32, wp.win - 2, s);- Xfer x;- x.src[0] = fc.pdst;- x.dst[0] = sr.stage(slot);- x.bytes[0] = (size_t) ng * ckv_rb();- x.n = 1;- if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {- x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();- x.dst[1] = sr.stage(slot) + x_cap() * ckv_rb();- x.bytes[1] = (size_t) ng * ik_rb();- x.n = 2;- }- xfer_put(x, sr.flag(slot), wp.win, d.xcnt, s);- }- }- ckv = sr.ckv[(size_t) l.dev];- }- // the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc- if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)- for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {- const int na = std::min(c.A, T - a0);- if (!gemm(d, c, l.wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.q, HQ, 0.0f, err)) return false;- Finish fr;- fr.rd = g.rd;- fr.pos = (int) (p0 + a0);- fr.vpp = NHL;- fr.freqs = freqs;- finish(c.q, na * NHL, HD, HD, fr, s);- if (a0 == 0 && na == T) dump_sub(d, "q", l.il, p0, T, c.q, (size_t) T * HQ * 4);- AttnRows ar;- ar.gen = 1;- ar.p0 = p0;- ar.t0 = a0;- ar.win = g.win;- ar.R = l.ratio;- ar.kcap = g.ix_topk;- sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,- na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)- if (a0 == 0 && na == T) dump_sub(d, "o", l.il, p0, T, c.o, (size_t) T * HQ * 4);- Finish fo;- fo.rd = g.rd;- fo.pos = (int) (p0 + a0);- fo.vpp = NHL;- fo.freqs = freqs;- fo.inverse = 1;- fo.quant = 1;- finish(c.o, na * NHL, HD, HD, fo, s);- if (a0 == 0 && na == T) dump_sub(d, "o_fin", l.il, p0, T, c.o, (size_t) T * HQ * 4);- x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);- const int per = HQ / OG;- for (int gi = 0; gi < OG; ++gi)- if (!gemm(d, c, l.wo_a[gi], c.o16 + (size_t) gi * per, HQ, na, c.oa + (size_t) gi * g.o_lora,- OG * g.o_lora, 0.0f, err))- return false;- x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);- if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;- if (a0 == 0 && na == T) dump_sub(d, "ao", l.il, p0, T, c.acc, (size_t) T * D * 4);- }- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);- dump_sub(d, "h_in", l.il, p0, T, c.h, (size_t) T * H * D * 4);- hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);- if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)- prof_mark(d, 1);- // the MoE- if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;- hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);- hc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);- x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);- gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)- for (int r0 = 0; r0 < T; r0 += kVlRows) { // (route()'s image bits cover kVlRows rows a call)- const int nr = std::min(kVlRows, T - r0);- VlBits vb;- const bool vl = l.gate_bias_vl != nullptr && vl_bits(p0 + r0, nr, vb);- route(c.glog + (size_t) r0 * 1024, l.gate_bias, l.n_exp, l.k_exp, nr, g.route_scale,- c.ridx + (size_t) r0 * kPicks, c.rw + (size_t) r0 * kPicks, s, l.gate_bias_vl, vl ? &vb : nullptr);- }- const int n_vs = L0.n_slots + (int) L0.host_list.size();- CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)- moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);- CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));- // the shared expert into acc while the host waits for the bounds- if (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))- return false;- swiglu(c.sg, c.su, T * FF, g.lim, c.sg, s);- x3::rows_f16(c.sg, FF, nullptr, T, FF, c.sh16, s);- if (!gemm(d, c, l.s2, c.sh16, FF, T, c.acc, D, 0.0f, err)) return false;- prof_mark(d, 2);- CK(cudaStreamSynchronize(s));- std::vector<int> b(c.bounds_pin, c.bounds_pin + n_vs + 1);- if (dump_r != nullptr && mi == 0) {- const size_t nk = (size_t) T * l.k_exp, np = (size_t) T * kPicks;- std::vector<int> ri(np), rt(nk);- std::vector<float> gl((size_t) T * l.n_exp), gb((size_t) l.n_exp);- CK(cudaMemcpy(ri.data(), c.ridx, np * 4, cudaMemcpyDeviceToHost));- CK(cudaMemcpy(rt.data(), c.row_tok, nk * 4, cudaMemcpyDeviceToHost));- CK(cudaMemcpy2D(gl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,- cudaMemcpyDeviceToHost));- CK(cudaMemcpy(gb.data(), l.gate_bias, gb.size() * 4, cudaMemcpyDeviceToHost));- std::lock_guard<std::mutex> lk(dump_mu);- const int64_t hdr[8] = {l.il, p0, T, l.k_exp, l.n_exp, n_vs, L0.n_slots, kPicks};- std::fwrite(hdr, 8, 8, dump_r);- std::fwrite(ri.data(), 4, np, dump_r);- std::fwrite(L0.vs_pin, 4, (size_t) l.n_exp, dump_r);- std::fwrite(b.data(), 4, b.size(), dump_r);- std::fwrite(rt.data(), 4, nk, dump_r);- std::fwrite(gl.data(), 4, gl.size(), dump_r);- std::fwrite(gb.data(), 4, gb.size(), dump_r);- std::fflush(dump_r);- }- if (mi == 0 && trace_f != nullptr && !l.mtp) { // DS4_TRACE: a chunk's picks, u8 (layer | 0x80), u16 T, T x k int16 ids- std::vector<int> ri((size_t) T * kPicks);- CK(cudaMemcpy(ri.data(), c.ridx, ri.size() * 4, cudaMemcpyDeviceToHost));- std::lock_guard<std::mutex> lk(dump_mu);- const uint8_t hd[3] = {(uint8_t) (l.il | 0x80), (uint8_t) (T & 255), (uint8_t) (T >> 8)};- std::fwrite(hd, 1, 3, trace_f);- for (int t = 0; t < T; ++t)- for (int i = 0; i < l.k_exp; ++i) {- const int16_t v = (int16_t) ri[(size_t) t * kPicks + i];- std::fwrite(&v, 2, 1, trace_f);- }- }- if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)- uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;- for (int e = 0; e < l.n_exp; ++e) {- const int v = L0.vs_pin[e];- pr[e] += (uint32_t) (b[(size_t) v + 1] - b[(size_t) v]);- }- st.experts += (int64_t) T * l.k_exp;- }- if (c.flow) {- if (!flow_layer(d, c, L0, mi, wp.k, T, b, err)) return false;- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);- hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);- CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));- prof_mark(d, 3);- return true;- }- // the streamed experts as their batches arrive (each batch's slots freed for the copies at once), the VRAM ones- // in shares between them (the device works while the copies run)- const int di = l.dev, P = Chunk::kParts;- int nb = 0;- for (size_t q = S.kp; q < S.b.size() && S.b[q].il == L0.il && S.b[q].k == wp.k; ++q) ++nb;- int v_done = 0;- for (int bi = 0; bi < nb; ++bi) {- const Batch& bt = S.b[S.kp];- const int half = (int) (S.kp % P);- CK(cudaStreamWaitEvent(s, c.filled[half], 0));- const int v0 = L0.n_slots + bt.j0;- if (!run_set(d, c, l, c.ring + (size_t) half * (c.ring_n / P) * c.ring_stride, v0, c.ring_stride, c.bounds,- b.data(), v0, v0 + bt.n, T, err))- return false;- CK(cudaEventRecord(c.freed[half], s));- ++S.kp;- if (!copy_ahead(d, c, S, di, err)) return false;- const int v_to = (int) ((int64_t) L0.n_slots * (bi + 1) / nb);- if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, v_to, T, err)) return false;- v_done = v_to;- }- if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;- if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);- hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);- CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));- prof_mark(d, 3);- return true;- }-- // the CPU lane's experts of every layer, in index order, and each expert's virtual slot (before a chunk)- void plan_streams() {- for (Layer& l : L) {- if (l.mtp || l.vs_pin == nullptr) continue;- l.host_list.clear();- for (int e = 0; e < l.n_exp; ++e)- if (l.slot[(size_t) e] < 0) l.host_list.push_back(e);- for (int m = 0; m < l.nm; ++m) { // (a group's members: each its own copy)- Layer& x = member(l, m);- int j = 0;- for (int e = 0; e < l.n_exp; ++e) x.vs_pin[e] = l.slot[(size_t) e] >= 0 ? l.slot[(size_t) e] : l.n_slots + j++;- }- }- }-- // a chunk's hand-off between groups: h | pre | topk | cand of a chunk of T rows- Xfer chunk_xfer(uint8_t* host, Chunk& c, int T, bool put) const {- const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,- nc = (size_t) T * cand_stride() * 4;- void* dv[4] = {c.h, c.pre, c.topk, c.cand};- uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};- const size_t nb[4] = {nh, np, nk, nc};- Xfer x;- x.n = 4;- for (int i = 0; i < 4; ++i) {- x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];- x.dst[i] = put ? (void*) hv[i] : dv[i];- x.bytes[i] = nb[i];- }- return x;- }-- // a run of chunks (start positions p0[k], sizes T[k], seqs seq[k]) on device di: its layers chunk after chunk, the- // groups pipelined (a group takes chunk k + 1 while the next takes chunk k: the hand-offs and the sources' rows- // two deep, each slot reused once its readers acknowledged it), the streamed experts' copies running ahead across- // layers and chunks- struct Run {- std::vector<int64_t> p0;- std::vector<int> T;- std::vector<unsigned> seq;- int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)- };- // layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):- // layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its- // layers and back before the next - so a layer's experts are read from the files once for the whole run (the- // layer cache keeps what a chunk read for the chunks after it), where chunk after chunk reads them once a chunk- bool run_dev_layers(int di, const Run& run, std::string& err) {- Dev& d = devs[(size_t) di];- Chunk& c = *chunks[(size_t) di];- cudaSetDevice(d.id);- cudaStream_t s = d.st;- const int D = g.dim, H = g.hc, nk = (int) run.T.size();- Streamer S; // (none: the flow streams)- const size_t hb = chunk_hand_bytes();- // the Engram rows: (layer, chunk) after (layer, chunk) in the run's order, each read on a helper thread while- // the one before runs (slot u % 2, once unit u - 2's copy of it is done)- std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)- for (int il : c.fl_layers)- if (L[(size_t) il].eg >= 0)- for (int k = 0; k < nk; ++k) egu.push_back({L[(size_t) il].eg, k});- const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);- auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {- if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);- const int li = egu[u].first, k = egu[u].second;- engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[u % 2] + (size_t) li * c.C * eg_per);- };- size_t eg_u = 0;- std::future<void> eg_next;- if (!egu.empty()) eg_next = std::async(std::launch::async, read_eg, (size_t) 0);- for (size_t li = 0; li < c.fl_layers.size(); ++li) {- const int il = c.fl_layers[li];- Layer& L0 = L[(size_t) il];- const bool last_l = li + 1 == c.fl_layers.size();- for (int k = 0; k < nk; ++k) {- const int64_t p0 = run.p0[(size_t) k];- const int T = run.T[(size_t) k], slot = k % 2;- const bool last_k = k + 1 == nk;- WinPlan wp;- wp.par = true;- wp.async = true;- wp.win = run.seq[(size_t) k];- wp.k = k;- wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;- if (L0.eg >= 0) { // this unit's rows ready, the next one's on their way- const double te = now_ms();- eg_next.get();- st.engram_ms += now_ms() - te;- wp.eg_slot = (int) (eg_u % 2);- if (++eg_u < egu.size()) eg_next = std::async(std::launch::async, read_eg, eg_u);- }- if (li == 0) { // the chunk's input: its embedding rows- if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));- for (int t = 0; t < T; ++t)- std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),- (size_t) D * 2);- CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));- CK(cudaEventRecord(c.emb_ev[slot], s));- embed(c.emb, T, c.h, D, H, s);- if (std::getenv("DS4_DBG_EMB")) {- ds4_metal_flush();- float *hp = c.h;- float hh[4] = {0,0,0,0};- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);- }- init_pre(c.pre, T, s);- } else { // or its state after the layer before- const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);- for (int i = 0; i < x.n; ++i)- CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyHostToDevice, s));- }- if (last_k && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);- if (it != g.d_targets.end()) {- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();- const int nr = std::min(T, tap_cap);- hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);- CK(cudaStreamWaitEvent(s, ev_seed, 0));- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,- (size_t) nr, cudaMemcpyDeviceToHost, s));- }- }- if (!layer_chunk(L0, 0, p0, T, wp, S, err)) return false;- if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);- if (!last_l) { // the chunk's state kept for the next layer- const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);- for (int i = 0; i < x.n; ++i)- CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyDeviceToHost, s));- continue;- }- // through every layer: the chunk's end, and the logits- if ((size_t) k < chunk_done.size()) {- CK(cudaEventRecord(chunk_done[(size_t) k], s));- chunk_rec[(size_t) k].store(1, std::memory_order_release);- }- if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time- const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;- for (int r0 = 0; r0 < T; r0 += kWin) {- const int nr = std::min(kWin, T - r0);- hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);- if (!head_share(d, d.mi, d.x, nr, err)) return false;- CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,- (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));- CK(cudaStreamSynchronize(s));- }- } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)- const int t0 = T - n_logits;- hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);- if (!head_share(d, d.mi, d.x, n_logits, err)) return false;- }- }- lm_layers.store((int) li + 1, std::memory_order_release);- }- return true;- }- bool run_dev_chunks(int di, const Run& run, std::string& err) {- Dev& d = devs[(size_t) di];- Chunk& c = *chunks[(size_t) di];- cudaSetDevice(d.id);- cudaStream_t s = d.st;- const int D = g.dim, H = g.hc, nk = (int) run.T.size();- Streamer S;- const int B = std::max(1, c.ring_n / Chunk::kParts);- if (c.flow) { // (the flow decides its streams as the layers route; nothing planned here)- CK(cudaStreamSynchronize(s));- CK(cudaStreamSynchronize(c.cs));- c.items.clear();- c.base = c.n_read = c.n_dma = c.n_comp = 0;- c.n_file = 0;- std::fill(c.stage_item.begin(), c.stage_item.end(), -1);- c.fl_layers.clear();- for (int il = 0; il < g.nl; ++il) // (a group's layers: on each member)- for (int m = 0; m < L[(size_t) il].nm; ++m)- if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);- c.fl_T = run.T;- c.spec_k = c.spec_il = -1;- c.wait_ms = 0;- c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;- c.lc_il = -1;- c.lc_used = 0;- c.lc_first = c.lc_chk = 0;- c.io_d0 = c.io->direct_bytes();- c.io_b0 = c.io->buffered_bytes();- if (!c.fl_layers.empty()) {- flow_spec(c, 0, c.fl_layers[0], run.T[0]);- if (!flow_pump(c, err)) return false;- }- if (c.lm) return run_dev_layers(di, run, err);- }- for (int k = 0; k < nk && !c.flow; ++k)- for (int il = 0; il < g.nl; ++il) {- const Layer& L0 = L[(size_t) il];- bool here = false;- for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;- if (!here) continue;- for (int j0 = 0; j0 < (int) L0.host_list.size(); j0 += B)- S.b.push_back({k, il, j0, std::min(B, (int) L0.host_list.size() - j0)});- }- if (!S.b.empty() && c.ring == nullptr) {- err = "no slots to stream experts into on CUDA" + std::to_string(d.id);- return false;- }- std::atomic<bool> ra_stop{false};- std::thread ra;- if (file_experts && !S.b.empty()) ra = std::thread([this, &S, di, &ra_stop] { read_ahead(S, di, ra_stop); });- struct Join { // (every way out of the run: the read-ahead ends before S does)- std::atomic<bool>& stop;- std::thread& th;- ~Join() {- stop.store(true);- if (th.joinable()) th.join();- }- } ra_join{ra_stop, ra};- if (!copy_ahead(d, c, S, di, err)) return false;- // this device's Engram layers: chunk k's rows read on a helper thread while chunk k - 1 runs (slot k % 2, once- // chunk k - 2's copy of it is done)- std::vector<int> egs;- for (const Layer& l0 : L)- for (int m = 0; m < l0.nm; ++m)- if (member(l0, m).dev == di && !l0.mtp && l0.eg >= 0) egs.push_back(l0.eg);- const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);- auto read_eg = [this, &c, &run, &egs, eg_per](int k) {- if (k >= 2) cudaEventSynchronize(c.eg_ev[k % 2]);- for (int li : egs)- engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[k % 2] + (size_t) li * c.C * eg_per);- };- std::future<void> eg_next;- if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);- for (int k = 0; k < nk; ++k) {- const int64_t p0 = run.p0[(size_t) k];- const int T = run.T[(size_t) k], slot = k % 2;- const bool last_k = k + 1 == nk;- if (!egs.empty()) { // chunk k's rows ready, chunk k + 1's on their way- const double te = now_ms();- eg_next.get();- if (d.mi == 0) st.engram_ms += now_ms() - te;- if (k + 1 < nk) eg_next = std::async(std::launch::async, read_eg, k + 1);- }- WinPlan wp;- wp.par = true;- wp.async = true;- wp.win = run.seq[(size_t) k];- wp.k = k;- wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;- bool started = false;- std::vector<char> got(srcs.size(), 0);- for (int il = 0; il < g.nl; ++il) {- Layer& L0 = L[(size_t) il];- int mi = -1;- for (int m = 0; m < L0.nm; ++m)- if (member(L0, m).dev == di) mi = m;- if (mi < 0) continue;- if (!started) { // the chunk's input: its embedding rows, or the previous group's hand-off- started = true;- if (il == 0) {- if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));- for (int t = 0; t < T; ++t)- std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),- (size_t) D * 2);- CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));- CK(cudaEventRecord(c.emb_ev[slot], s));- embed(c.emb, T, c.h, D, H, s);- if (std::getenv("DS4_DBG_EMB")) {- ds4_metal_flush();- float *hp = c.h;- float hh[4] = {0,0,0,0};- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);- }- init_pre(c.pre, T, s);- } else {- Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];- xfer_get(chunk_xfer(a.hand[slot], c, T, false), a.hand_flag[slot], wp.win, s);- set_flag(a.hand_ack + ((size_t) slot * 8 + mi) * 32, wp.win, s);- }- }- for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {- if (sidx < 0 || got[(size_t) sidx]) continue;- got[(size_t) sidx] = 1;- Src& sr = srcs[(size_t) sidx];- if (L[(size_t) sr.layer].grp == L0.grp) continue;- const int64_t g_first = p0 / sr.ratio;- const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);- Xfer x;- if (ng > 0 && sr.ckv[(size_t) di] != nullptr) {- x.src[x.n] = sr.stage(slot);- x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();- x.bytes[x.n++] = (size_t) ng * ckv_rb();- }- if (ng > 0 && sr.ik[(size_t) di] != nullptr) {- x.src[x.n] = sr.stage(slot) + x_cap() * ckv_rb();- x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();- x.bytes[x.n++] = (size_t) ng * ik_rb();- }- if (x.n > 0) xfer_get(x, sr.flag(slot), wp.win, s);- set_flag(sr.x_ack + ((size_t) slot * devs.size() + di) * 32, wp.win, s); // (taken, or none)- }- if (last_k && mi == 0 && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);- if (it != g.d_targets.end()) {- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();- const int nr = std::min(T, tap_cap);- hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);- CK(cudaStreamWaitEvent(s, ev_seed, 0));- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,- (size_t) nr, cudaMemcpyDeviceToHost, s));- }- }- if (!layer_chunk(L0, mi, p0, T, wp, S, err)) return false;- if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);- const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;- if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last- const int nnext = L[(size_t) il + 1].nm;- if (k >= 2) wait_flags(c.hand_ack + (size_t) slot * 8 * 32, nnext, 32, wp.win - 2, s);- xfer_put(chunk_xfer(c.hand[slot], c, T, true), c.hand_flag[slot], wp.win, d.xcnt, s);- }- }- if (di == head_dev && (size_t) k < chunk_done.size()) {- CK(cudaEventRecord(chunk_done[(size_t) k], s));- chunk_rec[(size_t) k].store(1, std::memory_order_release);- }- if (d.grp == devs[(size_t) head_dev].grp) { // the head group: each member its share of the logits- if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time- const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;- for (int r0 = 0; r0 < T; r0 += kWin) {- const int nr = std::min(kWin, T - r0);- hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);- if (!head_share(d, d.mi, d.x, nr, err)) return false;- if (di != head_dev) continue;- CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,- (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));- CK(cudaStreamSynchronize(s));- }- } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)- const int t0 = T - n_logits;- hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);- if (!head_share(d, d.mi, d.x, n_logits, err)) return false;- }- }- }- return true;- }-- // ---- a prompt's path: the flow or windows of 16- // this PC's speeds for pick_path (every device on the flow, the experts in place): the DMA from pinned staging to a- // GPU; the readers copying RAM into staging; the disk read by them (experts not in RAM); the CPU lane on experts in- // RAM and on ones it reads from the disk first - each timed once, a few experts' worth (DS4_PATH_PICK=0: always the- // flow, as before)- void calibrate_paths() {- if (chunk <= 0 || chunk_min != kFlowChunkMin || std::getenv("DS4_CHUNK_MIN") != nullptr) return;- if (const char* v = std::getenv("DS4_PATH_PICK"))- if (std::atoi(v) == 0) return;- int di = -1;- for (size_t i = 0; i < chunks.size() && di < 0; ++i)- if (chunks[i] != nullptr && chunks[i]->flow && chunks[i]->S >= 2) di = (int) i;- if (di < 0) return;- Chunk& c = *chunks[(size_t) di];- Dev& d = devs[(size_t) di];- cudaSetDevice(d.id);- const double t0 = now_ms();- // the DMA: staging -> the ring (once to warm up, once timed)- const size_t nb = std::min({(size_t) 256 << 20, (size_t) c.S * c.stage_stride, (size_t) c.ring_n * c.ring_stride});- double ms = 0;- for (int rep = 0; rep < 2; ++rep) {- const double a = now_ms();- if (cudaMemcpyAsync(c.ring, c.stage, nb, cudaMemcpyHostToDevice, c.cs) != cudaSuccess ||- cudaStreamSynchronize(c.cs) != cudaSuccess) {- cudaGetLastError();- return;- }- ms = now_ms() - a;- }- pm.dma = (double) nb / std::max(1e-3, ms / 1000.0);- // the readers: RAM -> staging (a buffer of its own, touched first), in as many jobs as they are- auto run_jobs = [&](std::vector<IoQueue::Job>& jobs, std::atomic<int>& left) {- const double a = now_ms();- left.store((int) jobs.size());- c.io->push(jobs.data(), (int) jobs.size());- while (left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);- return now_ms() - a;- };- {- const size_t part = std::min(nb, (size_t) 64 << 20) / 8 / 4096 * 4096;- std::vector<uint8_t> src(part * 8, 1);- std::vector<IoQueue::Job> jobs(8);- std::atomic<int> left{0};- std::atomic<bool> bad{false};- for (int i = 0; i < 8; ++i) {- jobs[(size_t) i].mem = src.data() + (size_t) i * part;- jobs[(size_t) i].dst = c.stage + (size_t) i * part;- jobs[(size_t) i].len = part;- jobs[(size_t) i].left = &left;- jobs[(size_t) i].bad = &bad;- }- run_jobs(jobs, left);- const double t1 = run_jobs(jobs, left), t2 = run_jobs(jobs, left);- pm.copy = (double) (part * 8) / std::max(1e-3, std::min(t1, t2) / 1000.0);- }- // experts not in RAM (none of their sampled pages resident), of the trunk layer with the most CPU-lane ones:- // up to min(8, staging slots) read by the readers as the flow reads them, then 8 others through the CPU lane- // three times (from the disk, then in RAM: the faster of two)- int best = -1, most = 0;- for (int il = 0; il < g.nl; ++il) {- int n = 0;- for (int e = 0; e < L[(size_t) il].n_exp; ++e) n += L[(size_t) il].slot[(size_t) e] < 0;- if (n > most) most = n, best = il;- }- if (best < 0) return;- const Layer& l = L[(size_t) best];- std::vector<int> cold;- const int n_read = std::min(8, c.S), n_cpu = 8;- for (int e = 0; e < l.n_exp && (int) cold.size() < n_read + n_cpu; ++e) {- if (l.slot[(size_t) e] >= 0 || (!l.ram.empty() && l.ram[(size_t) e] >= 0)) continue;- bool res = false;- for (int r = 0; r < 3; ++r) res = res || os::resident_sampled(mx_mat(l, e, r), mx_part_bytes(l, r), 4);- if (!res) cold.push_back(e);- }- if ((int) cold.size() >= n_read + n_cpu) {- std::vector<IoQueue::Job> jobs;- std::atomic<int> left{0};- std::atomic<bool> bad{false};- for (int i = 0; i < n_read; ++i) {- uint8_t* sb = c.stage + (size_t) i * c.stage_stride;- for (int r = 0; r < 3; ++r) {- IoQueue::Job j;- const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];- j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];- j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];- j.off = l.mx_off[r] + (uint64_t) cold[(size_t) i] * l.mx_stride[r];- j.len = mx_part_bytes(l, r);- j.dst = sb;- j.left = &left;- j.bad = &bad;- j.map = pk.at(l.mx_file[r], j.off);- jobs.push_back(j);- sb += flow_region(r);- }- }- const double rd = run_jobs(jobs, left);- if (!bad.load()) pm.disk = (double) n_read * (double) l.blob / std::max(1e-3, rd / 1000.0);- }- { // the CPU lane: entries of one token (as most of a window's are), from the disk (if any were cold), then RAM- const size_t first = (int) cold.size() >= n_read + n_cpu ? (size_t) n_read : 0;- std::vector<Ent> ents;- for (int i = 0; i < n_cpu; ++i) {- Ent en;- en.e = first + (size_t) i < cold.size() ? cold[first + (size_t) i] : -1;- if (en.e < 0) // (no cold ones: the first CPU-lane experts, wherever they are)- for (int e = 0, k = 0; e < l.n_exp && en.e < 0; ++e)- if (l.slot[(size_t) e] < 0 && k++ == i) en.e = e;- if (en.e < 0) break;- en.n = 1;- en.slot0 = i;- en.tok[0] = 0;- ents.push_back(en);- }- if (ents.empty()) return;- std::vector<float> x((size_t) g.dim);- for (size_t i = 0; i < x.size(); ++i) x[i] = 0.02f * std::sin(0.37f * (float) i);- const double bytes = (double) ents.size() * (double) l.blob;- double a = now_ms();- cpu_experts(l, x.data(), ents, (size_t) g.dim);- const double t_first = now_ms() - a;- double t_warm = 1e30;- for (int rep = 0; rep < 2; ++rep) {- a = now_ms();- cpu_experts(l, x.data(), ents, (size_t) g.dim);- t_warm = std::min(t_warm, now_ms() - a);- }- pm.cpu = bytes / std::max(1e-3, t_warm / 1000.0);- if (first > 0 || (int) cold.size() >= n_cpu) pm.cpu_cold = bytes / std::max(1e-3, t_first / 1000.0);- }- pm.on = pm.dma > 0 && pm.copy > 0 && pm.cpu > 0;- if (o.verbose) {- char dk[32] = "not measured", cc[32] = "not measured";- if (pm.disk > 0) std::snprintf(dk, sizeof dk, "%.2f GB/s", pm.disk / 1e9);- if (pm.cpu_cold > 0) std::snprintf(cc, sizeof cc, "%.2f GB/s", pm.cpu_cold / 1e9);- std::fprintf(stderr,- "ds4: prompts through the flow or windows by this PC's speeds: DMA %.1f GB/s, RAM to staging "- "%.1f GB/s, the disk %s; the CPU lane %.1f GB/s in RAM, %s from the disk (%.0f ms)\n",- pm.dma / 1e9, pm.copy / 1e9, dk, pm.cpu / 1e9, cc, now_ms() - t0);- }- }- // layer l's CPU-lane experts (VRAM holds the rest) and the chance a token picks a given one of them: the profile's- // share of the layer's picks they took, spread evenly over them, times pm.m (what the flow's runs found)- double pm_rate(const Layer& l, int& n_cold) const {- const uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;- double sum = 0, cold = 0;- n_cold = 0;- for (int e = 0; e < l.n_exp; ++e) {- sum += pr[e];- if (l.slot[(size_t) e] < 0) cold += pr[e], ++n_cold;- }- if (n_cold == 0) return 0.0;- const double share = sum > 0 && cold > 0 ? cold / sum : (double) n_cold / l.n_exp;- return pm.m * (double) l.k_exp * share / n_cold;- }- // the CPU-lane experts n tokens pick in the trunk's layers, expected (each layer's n_cold (1 - e^(-rate n)))- double pm_picks(int n) const {- double d = 0;- for (int il = 0; il < g.nl; ++il) {- int nc = 0;- const double a = pm_rate(L[(size_t) il], nc);- d += nc * (1.0 - std::exp(-a * n));- }- return d;- }- // a run of one chunk of n tokens on the flow picked `picked` CPU-lane experts: pm.m to what makes the expectation- // that (bisection over its log; a running mean of the log)- void pm_learn(int n, double picked) {- if (!pm.on || n < 2 || picked < 1) return;- const double m0 = pm.m;- double lo = std::log(1e-3), hi = std::log(1e3);- for (int it = 0; it < 40; ++it) {- pm.m = std::exp(0.5 * (lo + hi));- (pm_picks(n) < picked ? lo : hi) = 0.5 * (lo + hi);- }- const double obs = 0.5 * (lo + hi);- pm.m = std::exp(pm.n_m == 0 ? obs : 0.7 * std::log(m0) + 0.3 * obs);- ++pm.n_m;- }- // a prompt of n tokens that the flow could take (n >= chunk_min): true the flow, false windows of 16 - each path's- // time predicted from the bytes of CPU-lane experts it would move (the flow: each picked one once, n_cold (1 -- // e^(-rate n)) a layer, pm_rate) or compute (windows: each window's own picks), those in RAM (each layer's share:- // sampled pages of up to 32 of its CPU-lane experts, mincore / QueryWorkingSetEx; the RAM tier's) and those not at- // this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size- // (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted- // faster (by half, with less than kPickRes in RAM); said with --verbose- bool pick_path(int n) {- pm.last = -1;- if (!pm.on || n >= kPickMax) return true;- double f_res = 0, f_cold = 0, w = 0;- const int nw = n / kWin, rem = n % kWin;- for (int il = 0; il < g.nl; ++il) {- const Layer& l = L[(size_t) il];- int nc = 0;- const double a = pm_rate(l, nc);- if (nc == 0) continue;- int sampled = 0, res = 0, seen = 0;- const int step = std::max(1, nc / 32);- for (int e = 0; e < l.n_exp; ++e) {- if (l.slot[(size_t) e] >= 0 || seen++ % step != 0) continue;- bool r = !l.ram.empty() && l.ram[(size_t) e] >= 0;- if (!r) {- r = true;- for (int p = 0; p < 3 && r; ++p) r = os::resident_sampled(mx_mat(l, e, p), mx_part_bytes(l, p), 2);- }- ++sampled;- res += r;- }- const double share = sampled > 0 ? (double) res / sampled : 0.0, b = (double) l.blob;- auto u = [&](int t) { return nc * (1.0 - std::exp(-a * t)); };- f_res += share * u(n) * b;- f_cold += (1.0 - share) * u(n) * b;- w += ((double) nw * u(kWin) + (rem > 0 ? u(rem) : 0.0)) * b;- }- const double r_fr = std::min(pm.copy, pm.dma), r_fc = pm.disk > 0 ? std::min(pm.disk, pm.dma) : r_fr;- const double r_wc = pm.cpu_cold > 0 ? pm.cpu_cold : 1.0 / (1.0 / r_fc + 1.0 / pm.cpu);- const double t_flow = f_res / r_fr + f_cold / r_fc;- const double t_win = std::max(0.0, w - f_cold) / pm.cpu + f_cold / r_wc;- const int cls = pm_cls(n), n_win = (n + kWin - 1) / kWin;- // (windows: once some ran, what a window took on average times the windows the prompt needs - each pays the- // fixed costs of every layer on every GPU again, which a pass of the flow pays once)- const double pf = pm_k(cls) * t_flow, pw = pm.win_n > 0 ? pm.win_s * n_win : t_win;- // (the flow unless windows are predicted faster - in half its time at least, unless kPickRes of the CPU-lane- // experts' bytes are in RAM: the flow wins on every PC whose experts come from an SSD, where a wrong pick would- // cost the most; windows win where they are all in RAM)- const double res_share = f_res / std::max(1.0, f_res + f_cold);- const bool flow = !(pw < pf && (res_share >= kPickRes || 2.0 * pw <= pf));- pm.last = flow ? 1 : 0;- pm.last_cls = cls;- pm.last_nw = n_win;- pm.last_pred = flow ? t_flow : t_win;- if (o.verbose)- std::fprintf(stderr,- "ds4: a prompt of %d tokens through %s: predicted %.2f s, %s %.2f s (%.0f%% of its CPU-lane "- "experts' %.1f GB in RAM)\n",- n, flow ? "the flow" : "windows", flow ? pf : pw, flow ? "windows" : "the flow", flow ? pw : pf,- 100.0 * res_share, (f_res + f_cold) / 1e9);- return flow;- }- // a prompt's size class for pm.k: 2-31, 32-127, 128-511, 512 tokens and more- static int pm_cls(int n) { return n < 32 ? 0 : n < 128 ? 1 : n < 512 ? 2 : 3; }- // the flow's time factor for size class c: its own, else its nearest class's, else 1 (never the windows': theirs- // is mostly the fixed costs a window pays again and again)- double pm_k(int c) const {- for (int dc = 0; dc < 4; ++dc)- for (int cc : {c - dc, c + dc})- if (cc >= 0 && cc < 4 && pm.kn[0][cc] > 0) return pm.k[0][cc];- return 1.0;- }- // n tokens through windows of 16, the last one's logits_rows rows of logits- bool feed_windows(const int* tokens, int n, int logits_rows, std::string& err) {- logits_all = false;- for (int a = 0; a < n; a += kWin) {- const int T = std::min(kWin, n - a);- if (!forward(tokens + a, T, a + T == n ? logits_rows : 0, err)) return false;- }- return true;- }- // the picked path took ms: the flow's predictions for the prompt's size class scaled by what it really takes (taken- // / predicted), windows' time a window - running means- void path_done(double ms) {- if (pm.last < 0 || pm.last_pred <= 0) return;- if (pm.last == 0) {- const double w = ms / 1000.0 / std::max(1, pm.last_nw);- pm.win_s = pm.win_n == 0 ? w : 0.7 * pm.win_s + 0.3 * w;- ++pm.win_n;- } else {- const double ratio = std::min(8.0, std::max(0.25, ms / 1000.0 / pm.last_pred));- double& k = pm.k[0][pm.last_cls];- int64_t& n = pm.kn[0][pm.last_cls];- k = n == 0 ? ratio : 0.7 * k + 0.3 * ratio;- ++n;- }- pm.last = -1;- }- // a run layer-major? - one device holding every layer on the flow, several chunks, and what the chunks after the- // first would read from the files again more than the chunks' states moved to host memory and back at every layer- // (DS4_LAYER_MAJOR=0: never); then its host memory: the chunks' states (pinned, else pageable) and the layer cache,- // the largest layer's experts read from the files if RAM allows (ram_avail() less kRamMargin; DS4_LAYER_CACHE_MB- // sets it) - at least one expert, or not layer-major- bool lm_begin(const Run& run) {- static const char* env = std::getenv("DS4_LAYER_MAJOR");- if (devs.size() != 1 || run.T.size() < 2 || (env != nullptr && std::atoi(env) == 0)) return false;- Chunk& c = *chunks[0];- if (!c.flow) return false;- size_t need = 0, n_exp = 0, blob_max = 0;- double file_b = 0;- for (int il = 0; il < g.nl; ++il) {- const Layer& l = L[(size_t) il];- size_t ne = 0;- for (int e : l.host_list)- if (l.ram.empty() || l.ram[(size_t) e] < 0) ++ne;- const size_t stride = (l.blob + 63) & ~(size_t) 63;- need = std::max(need, ne * stride);- n_exp = std::max(n_exp, (size_t) l.n_exp);- blob_max = std::max(blob_max, stride);- file_b += (double) ne * (double) l.blob;- }- int64_t rows = 0;- for (int T : run.T) rows += T;- const size_t hb = chunk_hand_bytes(), state_b = run.T.size() * hb;- const double moved = 2.0 * (double) rows * ((double) hb / (double) chunk) * (double) g.nl;- if (need == 0 || (double) (run.T.size() - 1) * file_b <= moved) return false;- size_t lc_bytes = need;- if (const char* mb = std::getenv("DS4_LAYER_CACHE_MB")) lc_bytes = std::min(need, (size_t) std::atoll(mb) << 20);- else {- const uint64_t avail = ram_avail();- if (avail != ~uint64_t{0})- lc_bytes = std::min<uint64_t>(need, avail > kRamMargin + state_b ? avail - kRamMargin - state_b : 0);- }- if (lc_bytes < blob_max) return false;- void* st_p = nullptr;- c.lm_pinned = cudaHostAlloc(&st_p, state_b, cudaHostAllocPortable) == cudaSuccess;- if (!c.lm_pinned) {- cudaGetLastError();- st_p = os::anon_alloc(state_b, -1, 1);- if (st_p == nullptr) return false;- }- c.lc = (uint8_t*) os::anon_alloc(lc_bytes, -1, 1);- if (c.lc == nullptr) {- if (c.lm_pinned) cudaFreeHost(st_p);- else os::anon_free(st_p, state_b);- return false;- }- c.lm_state = (uint8_t*) st_p;- c.lm_state_bytes = state_b;- c.lc_bytes = lc_bytes;- c.lc_of.assign(n_exp, -1);- c.lc_writer.assign(n_exp, -1);- c.lm = true;- lm_layers.store(0);- return true;- }- void lm_end() {- for (size_t di = 0; di < chunks.size(); ++di) {- Chunk& c = *chunks[di];- if (!c.lm) continue;- cudaSetDevice(devs[di].id);- cudaStreamSynchronize(devs[di].st); // (nothing of the run's still reads or writes them: the device's- cudaStreamSynchronize(c.cs); // copies done, every read the run issued done - a failed run's too)- for (size_t i = c.base; i < c.n_read; ++i)- while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);- if (c.lm_pinned) cudaFreeHost(c.lm_state);- else os::anon_free(c.lm_state, c.lm_state_bytes);- os::anon_free(c.lc, c.lc_bytes);- c.lm_state = c.lc = nullptr;- c.lm_state_bytes = c.lc_bytes = 0;- c.lm = c.lm_pinned = false;- }- }-- // n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of- // the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)- bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {- if (n < 1) return true;- if (pos + n > o.max_context) {- err = "the context is full";- return false;- }- for (int t = 0; t < n; ++t)- if (tk[t] < 0 || tk[t] >= g.vocab) {- err = "token " + std::to_string(tk[t]) + " out of range";- return false;- }- const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;- if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);- Run run;- for (int a = 0; a < n; a += per) {- run.p0.push_back(pos + a);- run.T.push_back(std::min(per, n - a));- run.seq.push_back(++win_seq);- }- logits_all = logits_rows > kWin;- if (logits_all) chunk_logits.resize((size_t) n * g.vocab);- for (int t = 0; t < n; ++t) {- toks.push_back(tk[t]);- ctoks.push_back(ctok(pos + t, tk[t]));- }- plan_streams();- if (snap_slot >= 0) { // feed_mark: the state before the run's last row, kept by its layers on the way- run.snap_pos = pos + n - 1;- for (int il = 0; il < g.nl; ++il) {- Layer& l = L[(size_t) il];- Dev& d = devs[(size_t) l.dev];- if (l.snap_win == nullptr && !dalloc(d, l.snap_win, (size_t) g.win * win_rb(), err)) return false;- if (l.c_kv != nullptr && l.snap_kv == nullptr &&- (!dalloc(d, l.snap_kv, (size_t) l.ratio * g.hd, err) || !dalloc(d, l.snap_sc, (size_t) l.ratio * g.hd, err)))- return false;- }- }- const int T_last = run.T.back();- n_logits = logits_all ? 0 : std::min(logits_rows, T_last);- const double tl = now_ms();- { // each chunk's end on the head device, for the progress reports- cudaSetDevice(devs[(size_t) head_dev].id);- while (chunk_done.size() < run.T.size()) {- cudaEvent_t e = nullptr;- CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));- chunk_done.push_back(e);- }- chunk_rec.reset(new std::atomic<int>[chunk_done.size()]);- for (size_t i = 0; i < chunk_done.size(); ++i) chunk_rec[i].store(0);- }- prof_chunk = prof_on;- const bool lm = lm_begin(run);- struct LmEnd { // (every way out: its host memory freed once the device is done with it)- Impl* p;- bool on;- ~LmEnd() {- if (on) p->lm_end();- }- } lm_end_{this, lm};- std::vector<std::string> errs(devs.size());- for (size_t di = 0; di < devs.size(); ++di)- workers[di]->submit([this, di, &run, &errs] {- std::string e;- if (!run_dev_chunks((int) di, run, e)) errs[di] = e.empty() ? "a device's thread failed" : e;- });- size_t reported = 0;- for (;;) {- bool all = true;- for (auto& w : workers) all = all && w->idle();- if (progress && lm) { // (layer-major: the share of the layers every chunk went through)- const int nd = lm_layers.load(std::memory_order_acquire);- if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);- } else if (progress) { // chunks through every layer (the last layer's device past their event)- while (reported < run.T.size() && chunk_rec[reported].load(std::memory_order_acquire) &&- cudaEventQuery(chunk_done[reported]) == cudaSuccess)- ++reported;- if (reported > 0 && reported < run.T.size())- progress(run.p0[reported - 1] + run.T[reported - 1] - run.p0[0]);- }- if (all) break;- std::this_thread::sleep_for(std::chrono::milliseconds(50));- }- for (const std::string& e : errs)- if (!e.empty()) {- err = e;- return false;- }- st.layer_ms += now_ms() - tl;- eg_stats();- Dev& d = devs[(size_t) head_dev];- cudaSetDevice(d.id);- const double th = now_ms();- CK(cudaStreamSynchronize(d.st));- st.head_ms += now_ms() - th;- if (prof_chunk) { // DS4_PROF: the run's GPU time by stage, every device's summed- for (Dev& dv : devs) {- cudaSetDevice(dv.id);- CK(cudaStreamSynchronize(dv.st));- }- cudaSetDevice(d.id);- prof_collect();- std::fprintf(stderr,- "ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms "- "(with the waits for streamed ones), between layers %.0f ms\n",- n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);- for (double& v : cprof_ms) v = 0;- prof_chunk = false;- }- if (o.verbose || std::getenv("DS4_FLOW_STATS") != nullptr) {- const double ms = now_ms() - tl;- for (size_t di = 0; di < chunks.size(); ++di) {- const Chunk& c = *chunks[di];- if (!c.flow) continue;- const double db = (double) (c.io->direct_bytes() - c.io_d0) / 1e9;- const double bb = (double) (c.io->buffered_bytes() - c.io_b0) / 1e9;- const double fb = db + bb, rb = (double) c.it_ram * (double) L[0].blob / 1e9; // (a member: its parts)- char lcs[160] = "";- if (c.lm)- std::snprintf(lcs, sizeof lcs, ", %llu again from the layer cache (layer-major, %d chunks, %.1f GB "- "of RAM)", (unsigned long long) c.it_lc, (int) run.T.size(), (double) c.lc_bytes / 1e9);- std::fprintf(stderr,- "ds4 prompt stream, CUDA%d: %d tokens in %.0f ms; %llu experts streamed: %llu from the files "- "(%.1f GB, %.2f GB/s: %.1f GB from the disk unbuffered, %.1f GB through the page cache), %llu "- "from RAM (%.1f GB)%s; %llu likely ones queued early, %llu of them unpicked; the compute waited "- "%.0f ms for reads\n",- devs[di].id, n, ms, (unsigned long long) (c.it_file + c.it_ram + c.it_lc),- (unsigned long long) c.it_file, fb, fb / std::max(1e-3, ms / 1000.0), db, bb,- (unsigned long long) c.it_ram, rb, lcs, (unsigned long long) c.it_spec,- (unsigned long long) c.it_spec_unused, c.wait_ms);- }- }- if (pm.on && run.T.size() == 1) { // (the picks' spread for pick_path: the CPU-lane experts this run picked)- bool tp = false;- for (const Layer& l : L) tp = tp || l.nm > 1;- double picked = 0;- for (const auto& cp : chunks)- if (cp != nullptr && cp->flow) picked += (double) (cp->it_file + cp->it_ram) - (double) cp->it_spec_unused;- if (!tp) pm_learn(n, picked);- }- if (logits_all) n_logits = (T_last - 1) % kWin + 1; // (sample() takes the last row of d.logits)- pos += n;- st.tokens += n;- cache_tick(n);- const int nr = std::min(T_last, tap_cap);- if (!seed_drafter(pos - nr, nr, 0, err)) return false;- if (snap_slot >= 0) { // checkpoint `snap_slot` from what the layers kept (the drafter's blocks: as they are -- // the next feed seeds them again from its own rows, past this position)- Check& ckp = ck[snap_slot & 1];- ckp.pos = run.snap_pos;- ckp.win.resize(L.size());- ckp.comp.resize(L.size() * 2);- for (size_t i = 0; i < L.size(); ++i) {- Layer& l = L[i];- Dev& dv = devs[(size_t) l.dev];- cudaSetDevice(dv.id);- const bool kept = !l.mtp && l.snap_win != nullptr;- ckp.win[i].resize((size_t) g.win * win_rb());- CK(cudaMemcpyAsync(ckp.win[i].data(), kept ? l.snap_win : l.win, ckp.win[i].size(),- cudaMemcpyDeviceToHost, dv.st));- if (l.c_kv) {- ckp.comp[2 * i].resize((size_t) l.ratio * g.hd);- ckp.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);- CK(cudaMemcpyAsync(ckp.comp[2 * i].data(), kept && l.snap_kv ? l.snap_kv : l.c_kv,- ckp.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, dv.st));- CK(cudaMemcpyAsync(ckp.comp[2 * i + 1].data(), kept && l.snap_sc ? l.snap_sc : l.c_score,- ckp.comp[2 * i + 1].size() * 4, cudaMemcpyDeviceToHost, dv.st));- }- }- for (Dev& dv : devs) {- cudaSetDevice(dv.id);- CK(cudaStreamSynchronize(dv.st));- }- }- return true;- }-- // the window's plan, in window order (the indexer's and the candidate source's sizes carry from window to window)- void plan_window(int64_t p0, int T, WinPlan& wp) {- wp.par = par;- wp.async = par || (!sync_decode && T <= async_max_t);- wp.vl = 0;- for (int t = 0; t < T && !imgs.empty(); ++t)- if (in_image(p0 + t)) wp.vl |= 1u << t;- for (int il = 0; il < g.nl; ++il) {- const Layer& l = L[(size_t) il];- if (l.ratio > 0) {- const int R = l.ratio;- int n[kWin], k[kWin], clen_max = 0;- for (int t = 0; t < T; ++t) {- n[t] = (int) ((p0 + t + 1) / R);- k[t] = std::min(g.ix_topk, n[t]);- clen_max = std::max(clen_max, n[t]);- }- if (l.ix_src) {- for (int t = 0; t < T; ++t) wp.cand[il][t] = cur_cand_n[t];- if (clen_max > 0 && l.cand_src)- for (int t = 0; t < T; ++t) cur_cand_n[t] = std::min(g.cand_topk, (n[t] + g.cand_blk - 1) / g.cand_blk);- for (int t = 0; t < T; ++t) cur_topk_n[t] = k[t];- }- for (int t = 0; t < T; ++t) wp.topk[il][t] = cur_topk_n[t];- }- if (wp.async) {- wp.seq[il] = ++seq_issued;- layer_of[wp.seq[il] % kRing].store(il, std::memory_order_release);- }- }- }-- int head_nm() const { return (int) groups[(size_t) devs[(size_t) head_dev].grp].mem.size(); }- // the head's columns on member mi of its group of nm: an EXL3 head's split128 of the vocabulary (mat's), a GGUF's- // BF16 / Q8_0 head's even shares (mat_bf16's) - head_share's, its gather's and the markov steps' rows- void head_cols(int nm, int mi, int& v0, int& vn) const {- const Tensor* tw = pk.get("head.weight");- if (tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0")) {- vn = g.vocab / nm;- v0 = mi * vn;- } else- split128(g.vocab, nm, mi, v0, vn);- }- // the head over `rows` (<= 16) normed rows xn on member mi of the head group (device d): its share of each row's- // logits (columns [v0, v0 + vn), head_cols); the lead gathers the others' into d.logits (rows V apart) - two- // slots by seq (Vmax apart, the widest share's), a member reusing one once the lead acknowledged it. gather- // false: each member keeps its share (the drafter's markov steps over the group, mk_split)- bool head_share(Dev& d, int mi, const float* xn, int rows, std::string& err, bool gather = true) {- const int V = g.vocab, D = g.dim, nm = head_nm();- int v0 = 0, vn = 0, Vmax = 0, z = 0;- head_cols(nm, mi, v0, vn);- head_cols(nm, 0, z, Vmax);- Group& G = groups[(size_t) d.grp];- float* out = mi == 0 ? d.logits : d.logits_part;- const size_t ld = mi == 0 ? (size_t) V : (size_t) vn;- cudaSetDevice(d.id);- // (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4- // for a whole one, each pass reading the whole share; a GGUF's: 4 at a time, as before)- const int per = d.head.fmt != 0 ? 4 : std::max(1, x3::mv_cols(d.head)); // (8 columns a pass on Metal moved the KL by +0.002: the mvn8 kernel sums in another order)- for (int c = 0; c < rows; c += per)- if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))- return false;- if (nm == 1 || !gather) return true;- const unsigned seq = ++d.head_seq;- const int slot = (int) (seq % 2);- auto stage = [&](int m) { return G.lg_stage + ((size_t) slot * (nm - 1) + (m - 1)) * kWin * Vmax; };- auto flag = [&](int m) { return G.lg_flag + ((size_t) slot * (nm - 1) + (m - 1)) * 32; };- if (mi > 0) {- if (seq > 2) wait_flags(G.lg_ack + (size_t) slot * 32, 1, 32, seq - 2, d.st);- Xfer x;- x.n = 1;- x.src[0] = d.logits_part;- x.dst[0] = stage(mi);- x.bytes[0] = (size_t) rows * vn * 4;- xfer_put(x, flag(mi), seq, d.xcnt, d.st);- return true;- }- for (int m = 1; m < nm; ++m) {- int m0 = 0, mn = 0;- head_cols(nm, m, m0, mn);- Xfer x;- x.n = 1;- x.src[0] = stage(m);- x.dst[0] = d.logits_part;- x.bytes[0] = (size_t) rows * mn * 4;- xfer_get(x, flag(m), seq, d.st);- CK(cudaMemcpy2DAsync(d.logits + (size_t) m0, (size_t) V * 4, d.logits_part, (size_t) mn * 4,- (size_t) mn * 4, (size_t) rows, cudaMemcpyDeviceToDevice, d.st));- }- set_flag(G.lg_ack + (size_t) slot * 32, seq, d.st);- return true;- }-- // the head's rows (the last n_logits of the window) on head group member di (its h / pre: the group's)- bool head_rows_dev(int di, int T, std::string& err) {- Dev& d = devs[(size_t) di];- cudaSetDevice(d.id);- const int D = g.dim, H = g.hc;- if (n_logits <= 0) return true;- const int t0 = T - n_logits;- hc_pre_norm(d.h + (size_t) t0 * H * D, d.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, d.st);- return head_share(d, d.mi, d.x, n_logits, err);- }-- // the DSpark drafter's taps: the streams' mean entering target layer il, on device d (after the last seeding read- // h_taps)- bool tap(Dev& d, int il, int T, std::string& err) {- const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);- if (g.n_mtp == 0 || it == g.d_targets.end()) return true;- const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();- const int D = g.dim;- cudaSetDevice(d.id);- hc_mean(d.h, T, g.hc, D, d.tap, D, d.st);- CK(cudaStreamWaitEvent(d.st, ev_seed, 0));- CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4, (size_t) T,- cudaMemcpyDeviceToHost, d.st));- return true;- }-- bool forward(const int* tk, int T, int logits_rows, std::string& err, int seed_rows = -1) {-#if defined(STRATA_METAL)- struct BusyAll { // DS4_BUSY_ALL: the GPU is held busy for the whole step (the host gaps between its bursts too)- bool on;- BusyAll() : on(std::getenv("DS4_BUSY_ALL") != nullptr) { if (on) ds4_busy_begin(); }- ~BusyAll() { if (on) ds4_busy_end(); }- } busy_all;-#endif- if (T < 1 || T > kWin) {- err = "a window of " + std::to_string(T) + " tokens";- return false;- }- if (pos + T > o.max_context) {- err = "the context is full";- return false;- }- for (int t = 0; t < T; ++t)- if (tk[t] < 0 || tk[t] >= g.vocab) {- err = "token " + std::to_string(tk[t]) + " out of range";- return false;- }- const int64_t p0 = pos;- for (int t = 0; t < T; ++t) {- toks.push_back(tk[t]);- ctoks.push_back(ctok(pos + t, tk[t]));- }- // the Engram rows asked for now (the first Engram layer's first), collected right before their layer: what is- // left of their reads by then is the window's Engram time- const double te = now_ms();- const int64_t eg_w0 = eg_wait_ns.load();- for (int li = 0; li < pk.eg_layers; ++li) {- EgBatch& b = *eg_win[(size_t) li];- if (b.live) eg_finish(b); // (a window that failed midway: its reads end before h_eg is reused)- eg_issue(p0, T, li, nullptr, b);- }- st.engram_ms += now_ms() - te;- const int D = g.dim;- for (int t = 0; t < T; ++t) // (h_emb is written again by the next window, after this one's last sync)- std::memcpy(h_emb + (size_t) t * D, emb_row(p0 + t, tk[t]), (size_t) D * 2);- plan_window(p0, T, wplan);- prof_window = prof_on && T == 1;- n_logits = std::min(logits_rows, T);- const double tl = now_ms();- if (par) {- if (!forward_par(p0, T, err)) return false;- } else if (!forward_serial(p0, T, err)) return false;- st.layer_ms += now_ms() - tl;- for (auto& b : eg_win)- if (b->live) eg_finish(*b);- st.engram_ms += (double) (eg_wait_ns.load() - eg_w0) / 1e6;- eg_stats();- Dev& d = devs[(size_t) head_dev];- cudaSetDevice(d.id);- const double th = now_ms();- CK(cudaStreamSynchronize(d.st));- st.head_ms += now_ms() - th;- prof_collect();- pos += T;- st.tokens += T;- cache_tick(T);- if (seed_rows != 0 && !seed_drafter(p0, seed_rows < 0 ? T : seed_rows, 0, err)) return false;- return true;- }-- // one thread: the layers device after device, the hand-offs through pinned memory and events- void dbg_nf(const char* tag, int il, const float* dev, size_t n) { // DBG: non-finite count of a device buffer- ds4_metal_flush();- const float* hp = (const float*) strata::gpu::host_view(dev);- size_t nf = 0, first = 0; float mx = 0.0f;- for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) { if (!nf) first = i; ++nf; } else if (std::fabs(v) > mx) mx = std::fabs(v); }- std::fprintf(stderr, "DBG nf il=%d %s n=%zu nonfinite=%zu first=%zu max=%.4g\n", il, tag, n, nf, first, mx);- }- void dbg_h(const char* tag, Dev& d, int T) { // DBG: max |h| of the window's residual stream, after the queue drains- ds4_metal_flush();- const float* hp = (const float*) strata::gpu::host_view(d.h);- float mx = 0.0f; size_t nf = 0;- const size_t n = (size_t) T * g.hc * g.dim;- for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) ++nf; else if (std::fabs(v) > mx) mx = std::fabs(v); }- std::fprintf(stderr, "DBG hmax %s T=%d max=%.4g nonfinite=%zu h0=%g\n", tag, T, mx, nf, hp[0]);- }- bool forward_serial(int64_t p0, int T, std::string& err) {- const int D = g.dim, H = g.hc;- int cur = L[0].dev;- if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();- {- Dev& d = devs[(size_t) cur];- cudaSetDevice(d.id);- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));- embed(d.emb_rows, T, d.h, D, H, d.st);- if (std::getenv("DS4_DBG_EMB")) {- ds4_metal_flush();- float *hp = d.h;- float hh[4] = {0,0,0,0};- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);- }- init_pre(d.pre, T, d.st);- if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);- }- std::vector<float> dump;- if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);- for (int il = 0; il < g.nl; ++il) {- Layer& l = L[(size_t) il];- if (l.dev != cur) {- Dev& a = devs[(size_t) cur];- Dev& b = devs[(size_t) l.dev];- cudaSetDevice(a.id);- const size_t nh = (size_t) T * H * D * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,- nc = (size_t) T * cand_stride() * 4;- uint8_t* bb = a.bounce;- CK(cudaMemcpyAsync(bb, a.h, nh, cudaMemcpyDeviceToHost, a.st));- CK(cudaMemcpyAsync(bb + nh, a.pre, np, cudaMemcpyDeviceToHost, a.st));- CK(cudaMemcpyAsync(bb + nh + np, a.topk, nk, cudaMemcpyDeviceToHost, a.st));- CK(cudaMemcpyAsync(bb + nh + np + nk, a.cand, nc, cudaMemcpyDeviceToHost, a.st));- CK(cudaEventRecord(a.ev_out, a.st));- cudaSetDevice(b.id);- CK(cudaStreamWaitEvent(b.st, a.ev_out, 0));- CK(cudaMemcpyAsync(b.h, bb, nh, cudaMemcpyHostToDevice, b.st));- CK(cudaMemcpyAsync(b.pre, bb + nh, np, cudaMemcpyHostToDevice, b.st));- CK(cudaMemcpyAsync(b.topk, bb + nh + np, nk, cudaMemcpyHostToDevice, b.st));- CK(cudaMemcpyAsync(b.cand, bb + nh + np + nk, nc, cudaMemcpyHostToDevice, b.st));- cur = l.dev;- }- if (!tap(devs[(size_t) l.dev], il, T, err)) return false;- if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);- if (!layer(l, 0, p0, T, wplan, err)) return false;- if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers- Dev& dd = devs[(size_t) l.dev];- std::vector<float> hv((size_t) T * H * D);- cudaSetDevice(dd.id);- CK(cudaMemcpyAsync(hv.data(), dd.h, hv.size() * 4, cudaMemcpyDeviceToHost, dd.st));- CK(cudaStreamSynchronize(dd.st));- for (int t = 0; t < T; ++t)- std::memcpy(dump.data() + ((size_t) t * g.nl + l.il) * H * D, hv.data() + (size_t) t * H * D,- (size_t) H * D * 4);- }- }- if (dump_h != nullptr) std::fwrite(dump.data(), 4, dump.size(), dump_h);- if (cur != head_dev) {- err = "the head is not on the last layer's GPU";- return false;- }- return head_rows_dev(head_dev, T, err);- }-- // threads: each device's thread enqueues its layers (a group's members each their share) at once, the- // devices waiting on each other through host-mapped flags (the window's seq)- bool forward_par(int64_t p0, int T, std::string& err) {- wplan.win = ++win_seq;- std::vector<std::string> errs(devs.size());- for (size_t di = 0; di < devs.size(); ++di)- workers[di]->submit([this, di, p0, T, &errs] {- std::string e;- if (!run_dev((int) di, p0, T, wplan, e)) errs[di] = e.empty() ? "a device's thread failed" : e;- });- for (auto& w : workers) w->wait();- for (const std::string& e : errs)- if (!e.empty()) {- err = e;- return false;- }- return true;- }-- // the hand-off between groups: h | pre | topk | cand of device d <-> the host-mapped buffer- Xfer hand_xfer(uint8_t* host, Dev& d, int T, bool put) const {- const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,- nc = (size_t) T * cand_stride() * 4;- void* dv[4] = {d.h, d.pre, d.topk, d.cand};- uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};- const size_t nb[4] = {nh, np, nk, nc};- Xfer x;- x.n = 4;- for (int i = 0; i < 4; ++i) {- x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];- x.dst[i] = put ? (void*) hv[i] : dv[i];- x.bytes[i] = nb[i];- }- return x;- }-- bool run_dev(int di, int64_t p0, int T, const WinPlan& wp, std::string& err) {- Dev& d = devs[(size_t) di];- cudaSetDevice(d.id);- cudaStream_t s = d.st;- const int D = g.dim, H = g.hc;- bool started = false;- std::vector<char> got(srcs.size(), 0);- for (int il = 0; il < g.nl; ++il) {- Layer& L0 = L[(size_t) il];- int mi = -1;- for (int m = 0; m < L0.nm; ++m)- if (member(L0, m).dev == di) mi = m;- if (mi < 0) continue;- if (!started) { // the window's input: the embedding rows, or the previous group's hand-off- started = true;- if (il == 0) {- CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));- embed(d.emb_rows, T, d.h, D, H, s);- if (std::getenv("DS4_DBG_EMB")) {- ds4_metal_flush();- float *hp = d.h;- float hh[4] = {0,0,0,0};- cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);- std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);- }- init_pre(d.pre, T, s);- } else {- const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];- xfer_get(hand_xfer(a.hand, d, T, false), a.hand_flag, wp.win, s);- }- }- // the compressed KV / index keys another group's source layer wrote this window- for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {- if (sidx < 0 || got[(size_t) sidx]) continue;- got[(size_t) sidx] = 1;- Src& sr = srcs[(size_t) sidx];- if (L[(size_t) sr.layer].grp == L0.grp) continue;- const int64_t g_first = p0 / sr.ratio;- const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);- if (ng == 0) continue;- Xfer x;- if (sr.ckv[(size_t) di] != nullptr) {- x.src[x.n] = sr.x_stage;- x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();- x.bytes[x.n++] = (size_t) ng * ckv_rb();- }- if (sr.ik[(size_t) di] != nullptr) {- x.src[x.n] = sr.x_stage + x_cap() * ckv_rb();- x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();- x.bytes[x.n++] = (size_t) ng * ik_rb();- }- if (x.n > 0) xfer_get(x, sr.x_flag, wp.win, s);- }- if (mi == 0 && !tap(d, il, T, err)) return false;- if (!layer(L0, mi, p0, T, wp, err)) return false;- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, il, p0, T);- const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;- if (last && mi == 0 && il + 1 < g.nl) xfer_put(hand_xfer(d.hand, d, T, true), d.hand_flag, wp.win, d.xcnt, s);- }- if (d.grp == devs[(size_t) head_dev].grp) return head_rows_dev(di, T, err);- return true;- }-- // a source's new cache rows (src on device `from`, bytes of them) to the same place of its other devices' copies,- // through pinned memory: the copy out on the source's stream, each copy in on its device's stream after it- bool replicate(std::vector<uint8_t*>& copies, uint8_t* stage, cudaEvent_t ev, const uint8_t* src, size_t at,- size_t bytes, int from, std::string& err) {- bool any = false;- for (size_t od = 0; od < devs.size(); ++od) any = any || (copies[od] != nullptr && (int) od != from);- if (!any) return true;- Dev& d = devs[(size_t) from];- CK(cudaMemcpyAsync(stage, src, bytes, cudaMemcpyDeviceToHost, d.st));- CK(cudaEventRecord(ev, d.st));- for (size_t od = 0; od < devs.size(); ++od) {- if (copies[od] == nullptr || (int) od == from) continue;- Dev& o2 = devs[od];- cudaSetDevice(o2.id);- CK(cudaStreamWaitEvent(o2.st, ev, 0));- CK(cudaMemcpyAsync(copies[od] + at, stage, bytes, cudaMemcpyHostToDevice, o2.st));- }- cudaSetDevice(d.id);- return true;- }-- // layer L0 for the window, as group member mi (its share of the heads, of the output projection and of the FFNs;- // the partial sums all-reduced across the group)- bool layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, std::string& err) {- Layer& l = member(L0, mi);- Dev& d = devs[(size_t) l.dev];- cudaSetDevice(d.id);- cudaStream_t s = d.st;- const int nm = L0.nm;- const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;- const ArGroup& ag = groups[(size_t) L0.grp].ar;- const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;- if (std::getenv("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file's- ds4_metal_flush();- int shown = 0;- for (int e = 0; e < l.n_exp && shown < 3; ++e) {- if (l.slot[(size_t) e] < 0) continue;- std::vector<uint8_t> want(l.blob, 0);- read_mx_file(l, e, want.data());- const uint8_t* have = (const uint8_t*) strata::gpu::host_view(l.vram_base + (size_t) l.slot[(size_t) e] * unit(l));- size_t bad = 0, first = 0;- for (size_t i = 0; i < l.blob; ++i)- if (want[i] != have[i]) { if (!bad) first = i; ++bad; }- std::fprintf(stderr, "DBG slot il=%d e=%d slot=%d blob=%zu mismatching bytes=%zu first=%zu\n", l.il, e, l.slot[(size_t) e], l.blob, bad, first);- ++shown;- }- }- if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);- if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG- std::vector<float> hv((size_t) T * H * D);- cudaSetDevice(d.id);- ds4_metal_flush();- {- const float* hvp = (const float*) strata::gpu::host_view(d.h);- if (hvp) std::memcpy(hv.data(), hvp, hv.size() * 4);- }- int nf = 0, nzv = 0;- float mx = 0.0f;- size_t firstnan = 0, lastnan = 0;- for (size_t i = 0; i < hv.size(); ++i) {- const float v = hv[i];- if (!(v == v)) { if (nf == 0) firstnan = i; lastnan = i; ++nf; continue; }- if (v != 0.0f) ++nzv;- if (std::fabs(v) > mx) mx = std::fabs(v);- }- std::fprintf(stderr, "DBG h: il=%d dev=%d h=%p p0=%lld T=%d p0tok=%d zeros=%zu ninf=%d max=%.4g first=%.5g %.5g nan_range=%zu..%zu\n",- l.il, (int) l.dev, (void*) d.h, (long long) p0, T, toks.empty() ? -1 : toks[(size_t) p0],- hv.size() - (size_t) nzv - (size_t) nf, nf, mx, hv[0], hv.size() > 1 ? hv[1] : 0.0f, firstnan, lastnan);- }- prof_mark(d, 0);- // Engram- if (l.eg >= 0) {- const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;- const size_t per = (size_t) cols * (ehd + ehd / 32);- eg_finish(*eg_win[(size_t) l.eg]); // (asked for when the window began)- uint8_t* src = h_eg + (size_t) l.eg * eg_cap * per;- for (int t = 0; t < T; ++t) // [token][rows | scales] -> [all rows][all scales]- CK(cudaMemcpyAsync(d.eg_rows + (size_t) t * cols * ehd, src + (size_t) t * per, (size_t) cols * ehd,- cudaMemcpyHostToDevice, s));- uint8_t* sc = d.eg_rows + (size_t) T * cols * ehd;- for (int t = 0; t < T; ++t)- CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,- (size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));- engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);- dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);- dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);- if (std::getenv("DS4_DBG_EG")) {- dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);- const float* xv = (const float*) strata::gpu::host_view(d.eg_x);- for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)- if (!std::isfinite(xv[i])) {- const size_t t = i / ((size_t) cols * ehd), c = (i / ehd) % cols;- const uint8_t* dv = (const uint8_t*) strata::gpu::host_view(d.eg_rows) + (t * cols + c) * ehd;- const uint8_t* hv = src + t * per + c * ehd;- std::fprintf(stderr, "DBG eg bad x: t=%zu c=%zu e=%zu | dev/host row bytes:", t, c, i % ehd);- for (int b = 0; b < 80; ++b) std::fprintf(stderr, "%s%02x/%02x", b % 18 == 0 ? " |" : " ", dv[b], hv[b]);- std::fprintf(stderr, "\n");- break;- }- }- if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;- if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);- dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);- dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);- dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);- engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);- dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);- if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);- }- dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);- dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));- // attention- hc_mix(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);- hc_split(d.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, d.a_pre, d.a_post, d.a_comb, s);- dump_sub(d, "a_mix", l.il, p0, T, d.mix, (size_t) T * 32 * 4);- dump_sub(d, "a_pre", l.il, p0, T, d.a_pre, (size_t) T * 8 * 4);- dump_sub(d, "a_post", l.il, p0, T, d.a_post, (size_t) T * 8 * 4);- dump_sub(d, "a_comb", l.il, p0, T, d.a_comb, (size_t) T * 64 * 4);- hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);- dump_sub(d, "pre_in", l.il, p0, T, d.pre, (size_t) T * 8 * 4);- dump_sub(d, "attn_x", l.il, p0, T, d.x, (size_t) T * D * 4);- {- x3::MvJob j[4];- int nj = 0;- j[nj++] = job(l.wq_a, d.x, d.qr, T, D, g.q_lora);- j[nj++] = job(l.wkv, d.x, d.kvr, T, D, HD);- if (l.kv_src) j[nj++] = job(l.c_wkv, d.x, d.ckr, T, D, HD);- if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);- if (!mv_n(d, j, nj, err)) return false;- }- dump_sub(d, "qr_raw", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);- dump_sub(d, "kvr_raw", l.il, p0, T, d.kvr, (size_t) T * HD * 4);- Finish fq;- fq.w = l.q_norm;- fq.eps = g.eps;- finish(d.qr, T, g.q_lora, g.q_lora, fq, s);- dump_sub(d, "qr", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);- if (NHL > 0) {- if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;- Finish fr;- fr.rd = g.rd;- fr.pos = (int) p0;- fr.vpp = NHL;- fr.freqs = freqs;- finish(d.q, T * NHL, HD, HD, fr, s);- }- if (NHL > 0) dump_sub(d, "q", l.il, p0, T, d.q, (size_t) T * HQ * 4);- // the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark- // block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of- // that ring and to the whole block, and leaves the ring as it was- ring_to_ext(l.win, d.ext, l.mtp ? (int64_t) g.win : p0, g.win, HD, s); // (a block's ring: its rows as they lie)- Finish fk;- fk.w = l.kv_norm;- fk.eps = g.eps;- fk.rd = g.rd;- fk.pos = (int) p0;- fk.freqs = freqs;- fk.quant = 1;- fk.dst = d.ext + (size_t) g.win * HD;- fk.dst_stride = HD;- finish(d.kvr, T, HD, HD, fk, s);- if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);- dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);- dump_sub(d, "ext", l.il, p0, T, d.ext, ((size_t) g.win + T) * HD * 2); // (BF16 rows: the ring's, then this window's)- AttnRows ar;- for (int t = 0; t < T; ++t) {- if (l.mtp) {- ar.win0[t] = 0;- ar.nwin[t] = (int) std::min<int64_t>(g.win, p0);- ar.nsel[t] = T;- continue;- }- const int first = (int) std::max<int64_t>(g.win - p0, t + 1); // ext row of position max(0, p - win + 1)- ar.win0[t] = first;- ar.nwin[t] = g.win + t - first + 1;- ar.nsel[t] = 0;- }- const void* ckv = nullptr;- int ckv_fmt = kKvFp4;- if (l.ratio > 0) {- Src& sr = srcs[(size_t) l.src];- const int R = l.ratio;- // the groups this window completes: g_first .. g_first + ng - 1- const int64_t g_first = p0 / R; // the first group whose last position is >= p0- const int64_t g_end = (p0 + T) / R; // groups completed by p0 + T - 1- const int ng = (int) std::max<int64_t>(0, g_end - g_first);- if (l.kv_src) {- if (R == 1) {- Finish fc;- fc.w = l.c_norm;- fc.eps = g.eps;- finish(d.ckr, T, HD, HD, fc, s);- CK(cudaMemcpyAsync(d.latent, d.ckr, (size_t) T * HD * 4, cudaMemcpyDeviceToDevice, s));- } else {- if (spec_win && l.c_snap != nullptr) { // to replay the accepted rows if drafts are rejected- const size_t ns = (size_t) R * HD * 4, nr = (size_t) T * HD * 4;- CK(cudaMemcpyAsync(l.c_snap, l.c_kv, ns, cudaMemcpyDeviceToDevice, s));- CK(cudaMemcpyAsync(l.c_snap + (size_t) R * HD, l.c_score, ns, cudaMemcpyDeviceToDevice, s));- CK(cudaMemcpyAsync(l.c_rows, d.ckr, nr, cudaMemcpyDeviceToDevice, s));- CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));- }- compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);- dump_sub(d, "ckr", l.il, p0, T, d.ckr, (size_t) T * HD * 4);- dump_sub(d, "cgr", l.il, p0, T, d.cgr, (size_t) T * HD * 4);- }- }- Rows lim;- int clen_max = 0;- for (int t = 0; t < T; ++t) {- lim.n[t] = (int) ((p0 + t + 1) / R);- lim.k[t] = std::min(g.ix_topk, lim.n[t]);- clen_max = std::max(clen_max, lim.n[t]);- }- if (l.ix_src) {- Src& ks = srcs[(size_t) l.ik_src];- if (l.owns_k && ng > 0) {- if (!mv(d, {job(l.i_wk, d.latent, d.ikr, ng, HD, g.ix_hd)}, err)) return false;- Finish fi;- fi.w = l.ik_norm;- fi.eps = g.eps;- fi.rd = g.rd;- fi.pos = (int) (g_first * R);- fi.pos_step = R;- fi.freqs = freqs;- fi.quant = 3;- fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();- fi.pdst_stride = (long long) ik_rb();- finish(d.ikr, ng, g.ix_hd, g.ix_hd, fi, s);- if (!wp.par && !replicate(ks.ik, ks.h_ik, ks.ev_ik, fi.pdst, (size_t) g_first * ik_rb(),- (size_t) ng * ik_rb(), l.dev, err))- return false;- }- if (clen_max > 0) {- if (!mv(d, {job(l.i_wq_b, d.qr, d.iq, T, g.q_lora, (size_t) g.ix_heads * g.ix_hd)}, err)) return false;- Finish fi;- fi.rd = g.rd;- fi.pos = (int) p0;- fi.vpp = g.ix_heads;- fi.freqs = freqs;- fi.quant = 3;- finish(d.iq, T * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);- // (iw rows are 64 apart)- gemv_f16_rows(l.i_wproj, d.x, (size_t) D, g.ix_heads, D, T,- 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), d.iw, 64, s);- const long long ss = (long long) o.max_context + 64;- bool restrict_ = false;- for (int t = 0; t < T; ++t)- restrict_ = restrict_ || (lim.n[t] + g.cand_blk - 1) / g.cand_blk > g.cand_topk;- if (l.uses_cand && restrict_) {- Rows cl; // the candidate positions of each query: its blocks x blk- int nmax = 0;- for (int t = 0; t < T; ++t) {- cl.n[t] = lim.n[t];- cl.k[t] = wp.cand[l.il][t];- nmax = std::max(nmax, wp.cand[l.il][t] * g.cand_blk);- }- index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, nmax, d.cand,- (int) cand_stride(), g.cand_blk, cl, d.scores, ss, s);- Rows tr;- for (int t = 0; t < T; ++t) {- tr.n[t] = wp.cand[l.il][t] * g.cand_blk;- tr.k[t] = lim.k[t];- }- topk(d.scores, ss, T, tr, d.cand, (int) cand_stride(), g.cand_blk, d.topk, (int) topk_stride(), s);- } else {- index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, clen_max, nullptr, 0, 1,- lim, d.scores, ss, s);- if (l.cand_src) {- Rows cr;- for (int t = 0; t < T; ++t) {- cr.n[t] = lim.n[t];- cr.k[t] = g.cand_topk;- }- cand_blocks(d.scores, ss, T, cr, g.cand_blk, d.bm, (long long) bm_stride(), d.cand,- (int) cand_stride(), s);- }- topk(d.scores, ss, T, lim, nullptr, 0, 1, d.topk, (int) topk_stride(), s);- }- }- }- if (l.kv_src && ng > 0) {- Finish fc;- fc.rd = g.rd;- fc.pos = (int) (g_first * R);- fc.pos_step = R;- fc.freqs = freqs;- fc.quant = 2;- fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();- fc.pdst_stride = (long long) ckv_rb();- finish(d.latent, ng, HD, HD, fc, s);- if (!wp.par && !replicate(sr.ckv, sr.h_ckv, sr.ev_ckv, fc.pdst, (size_t) g_first * ckv_rb(),- (size_t) ng * ckv_rb(), l.dev, err))- return false;- if (wp.par && mi == 0 && sr.remote) { // the new rows for the other groups' readers- Xfer x;- x.src[0] = fc.pdst;- x.dst[0] = sr.x_stage;- x.bytes[0] = (size_t) ng * ckv_rb();- x.n = 1;- if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {- x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();- x.dst[1] = sr.x_stage + x_cap() * ckv_rb();- x.bytes[1] = (size_t) ng * ik_rb();- x.n = 2;- }- xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);- }- }- if (ng > 0) dump_sub(d, "latent", l.il, p0, ng, d.latent, (size_t) ng * HD * 4);- dump_sub(d, "topk", l.il, p0, T, d.topk, (size_t) T * topk_stride() * 4);- for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;- ckv = sr.ckv[(size_t) l.dev];- }- if (l.mtp) { // the block's rows (BF16), after the ring's in ext, as every query's selection- ckv = d.ext + (size_t) g.win * HD;- ckv_fmt = kKvBf16;- iota_rows(d.topk, T, T, (int) topk_stride(), s);- }- if (NHL > 0) {- sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,- 1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);- dump_sub(d, "o", l.il, p0, T, d.o, (size_t) T * HQ * 4);- Finish fo;- fo.rd = g.rd;- fo.pos = (int) p0;- fo.vpp = NHL;- fo.freqs = freqs;- fo.inverse = 1;- fo.quant = 1;- finish(d.o, T * NHL, HD, HD, fo, s);- dump_sub(d, "o_fin", l.il, p0, T, d.o, (size_t) T * HQ * 4);- x3::MvJob j[16];- const int per = HQ / OG;- for (int gi = 0; gi < OG; ++gi)- j[gi] = job(l.wo_a[gi], d.o + (size_t) gi * per, d.oa + (size_t) gi * g.o_lora, T, HQ, (size_t) OG * g.o_lora);- if (!mv_n(d, j, OG, err)) return false;- if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;- } else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)- dump_sub(d, "ao", l.il, p0, T, d.ao, (size_t) T * D * 4);- if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);- hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);- std::swap(d.h, d.h2);- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)- dump_sub(d, "h_attn", l.il, p0, T, d.h, (size_t) T * H * D * 4);- prof_mark(d, 1);- // the MoE- hc_mix(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, s);- hc_split(d.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, d.f_pre, d.f_post, d.f_comb, s);- hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);- dump_sub(d, "f_pre", l.il, p0, T, d.f_pre, (size_t) T * 8 * 4);- dump_sub(d, "f_post", l.il, p0, T, d.f_post, (size_t) T * 8 * 4);- dump_sub(d, "f_comb", l.il, p0, T, d.f_comb, (size_t) T * 64 * 4);- dump_sub(d, "ffn_x", l.il, p0, T, d.x, (size_t) T * D * 4);- if (wp.async) {- // the routing goes to the service thread through the mailbox (a group's lead publishes it, the members- // make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the- // router made), and waits for the answer only before the experts' sum- gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);- const unsigned sq = wp.seq[l.il];- const int slot = (int) (sq % kRing);- const bool lead = mi == 0;- int na[2] = {0, 0}; // lookahead: the next layers' guesses (their gate on this FFN input), published below- for (int a = 1; lead && a <= std::min(2, ahead_mode); ++a) {- const Layer* nx = ahead_layer(L0, a);- if (nx == nullptr) break;- const int m = a == 1 ? std::max(1, std::min(kAhead1, nx->k_exp + ahead_extra)) : std::min(kPicks, nx->k_exp);- float* gl = d.glog_a + (size_t) (a - 1) * kWin * 1024;- gemv_f32(nx->gate_w, d.x, nx->n_exp, D, T, gl, 1024, s); // (normed with the next layer's FFN- // weights instead: the same hit rate)- route_ahead(gl, nx->gate_bias, nx->n_exp, m, T, reqs + slot, (a - 1) * kAhead1, s);- na[a - 1] = m;- }- route_publish(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, l.vram_tab, d.x, D, d.ridx, d.rw,- d.plan_ptr, lead ? reqs + slot : nullptr, sq, s, l.gate_bias_vl, l.mtp ? 0u : wp.vl, na[0],- na[1]);- if (!mv(d, {job(l.s1, d.x, d.sg, T, D, FF), job(l.s3, d.x, d.su, T, D, FF)}, err)) return false;- swiglu(d.sg, d.su, T * FF, g.lim, d.sh, s);- if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, FF, D)}, err)) return false;- prof_mark(d, 2);- const x3::ExpertLayout& lay = nm > 1 ? l.hlay : l.lay;- if (T == 1 && moe_token_path) {- x3::MoeArgs ma;- ma.plan_ptr = d.plan_ptr;- ma.plan_w = d.rw;- ma.cpu_part = lead ? resps[slot].part[0] : nullptr;- ma.cpu_flag = lead ? &resps[slot].cpu[0] : nullptr;- ma.lay = lay;- ma.k = l.k_exp;- ma.n_embd = D;- ma.n_ff = FF;- ma.limit = g.lim;- const bool mx = lay.fmt == gg::kFmtMXFP4;- if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)- zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);- if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);- else x3::moe_gate_up(ma, d.x, d.xg, s);- prof_mark(d, 3);- if (lead) wait_seq(&resps[slot].seq, sq, s);- prof_mark(d, 4);- if (mx) gg::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);- else x3::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);- } else { // each distinct expert read once for the window's tokens- gf::MoeWin mw;- mw.plan_ptr = d.plan_ptr;- mw.T = T;- mw.k = l.k_exp;- mw.grp_ptr = d.grp_ptr;- mw.grp_start = d.grp_start;- mw.grp_ent = d.grp_ent;- mw.n_groups = d.n_groups;- gf::moe_win_group(mw, s);- x3::WinArgs wa;- wa.grp_ptr = d.grp_ptr;- wa.grp_start = d.grp_start;- wa.grp_ent = d.grp_ent;- wa.n_groups = d.n_groups;- wa.plan_ptr = d.plan_ptr;- wa.plan_w = d.rw;- wa.cpu_part = lead ? &resps[slot].part[0][0] : nullptr;- wa.cpu_stride = kMaxDim;- wa.cpu_flag = lead ? resps[slot].cpu : nullptr;- wa.wait_ptr = lead ? &resps[slot].seq : nullptr;- wa.wait_val = sq;- wa.x = d.x;- wa.x_stride = (size_t) D;- wa.h = d.xg;- wa.part = d.ed;- wa.sh_out = d.sh_out;- wa.out = d.moe_out;- wa.out_stride = (size_t) D;- wa.lay = lay;- wa.T = T;- wa.k = l.k_exp;- wa.n_embd = D;- wa.n_ff = FF;- wa.limit = g.lim;- if (!(lay.fmt == gg::kFmtMXFP4 ? gg::win_experts(wa, s) : x3::win_experts(wa, s))) {- err = "exl3 win_experts failed";- return false;- }- }- if (lead) st.experts += (int64_t) T * l.k_exp;- if (nm > 1) allreduce(d.moe_out, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);- hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);- std::swap(d.h, d.h2);- CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));- prof_mark(d, 5);- return true;- }- if (nm > 1) {- err = "a tensor-parallel layer on the host-routed path";- return false;- }- // (glog rows are 1024 apart: route reads them so)- static const bool phases = std::getenv("DS4_PHASES") != nullptr; // debugging: where a decode layer's host time goes- static double ph[8] = {0}, ph_prev = 0; static long ph_n = 0;- const double ph0 = phases ? now_ms() : 0.0;- gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);- VlBits vb = {};- vb.w[0] = l.mtp ? 0u : wp.vl;- route(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, d.ridx, d.rw, s, l.gate_bias_vl,- vb.w[0] != 0 ? &vb : nullptr);- CK(cudaMemcpyAsync(h_idx, d.ridx, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));- CK(cudaMemcpyAsync(h_w, d.rw, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));- CK(cudaMemcpyAsync(h_x, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, s));- CK(cudaEventRecord(d.ev, s));- // the shared expert while the host looks at the picks- if (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;-#if defined(STRATA_METAL)- int la_n = 0; // the next layers' guessed picks (their gate on this FFN input), on the GPU: a gemv and a route a layer- if (T == 1 && la_enabled()) {- for (int a = 1; a <= la_depth(); ++a) {- const int il = l.il + a;- if (il >= g.nl) break;- const Layer& nx = L[(size_t) il];- if (nx.nm != 1 || nx.lay.fmt != gg::kFmtMXFP4 || nx.gate_bias == nullptr || nx.gate_w == nullptr) break;- gemv_f32(nx.gate_w, d.x, nx.n_exp, D, 1, d.glog_a + (size_t) (a - 1) * kWin * 1024, 1024, s);- route(d.glog_a + (size_t) (a - 1) * kWin * 1024, nx.gate_bias, nx.n_exp, nx.k_exp, 1, g.route_scale, d.ridx_a + (a - 1) * kPicks,- d.rw_a + (a - 1) * kPicks, s, nullptr, nullptr);- CK(cudaMemcpyAsync(h_idx_a + (a - 1) * kPicks, d.ridx_a + (a - 1) * kPicks, (size_t) kPicks * 4, cudaMemcpyDeviceToHost, s));- la_n = a;- }- if (la_n > 0) CK(cudaEventRecord(d.ev2, s));- }-#endif- if (std::getenv("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);- std::fprintf(stderr, "DBG s1 fmt=%d K=%d k=%d n=%d ld=%d raw=%p\n", l.s1.fmt, l.s1.K, l.s1.k, l.s1.n, l.s1.ld, (void*) l.s1.raw); }- swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);- if (std::getenv("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);- if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;- const double tw = now_ms();- CK(cudaEventSynchronize(d.ev));- st.gpu_wait_ms += now_ms() - tw;- const double ph2 = phases ? now_ms() : 0.0;- dump_sub(d, "glog", l.il, p0, T, d.glog, (size_t) T * 1024 * 4);- dump_sub_host("ridx", l.il, p0, T, h_idx, (size_t) T * kPicks * 4);- dump_sub_host("rw", l.il, p0, T, h_w, (size_t) T * kPicks * 4);- dump_sub(d, "sh_out", l.il, p0, T, d.sh_out, (size_t) T * D * 4);- // the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the- // CPU lane's entries- Combine cm;- Gather ga;- uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;- int vex[kWin * kPicks], vrow0[kWin * kPicks], vn[kWin * kPicks];- int nvex = 0, nrows = 0;- std::vector<Ent> ents;- std::vector<std::pair<int, float>> cpu_rows; // slot -> (token, weight)- std::vector<std::vector<std::pair<int, int>>> per((size_t) l.n_exp); // expert -> (token, pick)- for (int t = 0; t < T; ++t) {- cm.n[t] = 0;- cm.cpu[t] = 0;- for (int i = 0; i < l.k_exp; ++i) {- const int e = h_idx[t * kPicks + i];- if (e < 0 || e >= l.n_exp) {- err = "router picked expert " + std::to_string(e) + " (layer " + std::to_string(l.il) + ", token " + std::to_string(t) + ", pick " + std::to_string(i) + ", of " + std::to_string(l.k_exp) + ")";- return false;- }- ++pr[e];- l.heat[(size_t) e] += 1.0f;- if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);- per[(size_t) e].push_back({t, i});- }- }- int nslots = 0;- for (int e = 0; e < l.n_exp; ++e) {- const auto& pe = per[(size_t) e];- if (pe.empty()) continue;-#if defined(STRATA_METAL)- const bool on_gpu = l.slot[(size_t) e] >= 0 || (T == 1 && zc_try(l, e));-#else- const bool on_gpu = l.slot[(size_t) e] >= 0;-#endif- if (on_gpu) {- vex[nvex] = e;- vrow0[nvex] = nrows;- vn[nvex] = (int) pe.size();- for (const auto& tp : pe) {- ga.src[nrows] = tp.first;- const int t = tp.first;- cm.row[t][cm.n[t]] = nrows;- cm.w[t][cm.n[t]] = h_w[t * kPicks + tp.second];- ++cm.n[t];- ++nrows;- }- ++nvex;- } else {- for (size_t a = 0; a < pe.size(); a += kc::kExl3CpuMaxN) {- Ent en;- en.e = e;- en.slot0 = nslots;- en.n = (int) std::min<size_t>(kc::kExl3CpuMaxN, pe.size() - a);- for (int i = 0; i < en.n; ++i) {- en.tok[i] = pe[a + (size_t) i].first;- cpu_rows.push_back({pe[a + (size_t) i].first, h_w[pe[a + (size_t) i].first * kPicks + pe[a + (size_t) i].second]});- cm.cpu[en.tok[i]] = 1;- }- nslots += en.n;- ents.push_back(en);- }- }- }- st.experts += (int64_t) T * l.k_exp;- st.cpu_experts += nslots;-#if defined(STRATA_METAL)- if (T == 1) lookahead_prefetch(l, d.ev2, la_n, ents);-#endif- if (nrows > 0) {- gather_rows(d.x, ga, nrows, D, d.xe, s);- dump_sub(d, "xe", l.il, p0, nrows, d.xe, (size_t) nrows * D * 4);- { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }- dump_sub_host("vmap", l.il, p0, nvex, vmap.data(), vmap.size() * 4); }- x3::MvJob j[2 * kWin * kPicks];- int nj = 0;- for (int i = 0; i < nvex; ++i) {- const auto& vm = l.vm[(size_t) vex[i]];- j[nj++] = job(vm[0], d.xe + (size_t) vrow0[i] * D, d.xg + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);- j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);- }- if (!mv_n(d, j, nj, err)) return false;- dump_sub(d, "xg", l.il, p0, nrows, d.xg, (size_t) nrows * g.ff * 4);- dump_sub(d, "xu", l.il, p0, nrows, d.xu, (size_t) nrows * g.ff * 4);- swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);- dump_sub(d, "xhh", l.il, p0, nrows, d.xhh, (size_t) nrows * g.ff * 4);- nj = 0;- for (int i = 0; i < nvex; ++i)- j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,- vn[i], g.ff, D);- if (!mv_n(d, j, nj, err)) return false;- dump_sub(d, "ed", l.il, p0, nrows, d.ed, (size_t) nrows * D * 4);- }- const double ph3 = phases ? now_ms() : 0.0;- if (!ents.empty()) {- const double tc = now_ms();-#if defined(STRATA_METAL)- ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)- ds4_busy_begin(); // (and the GPU is kept out of its idle clocks until the lane is done)-#endif- static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)- if (!skip_cpu) cpu_experts(l, h_x, ents);- else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)-#if defined(STRATA_METAL)- ds4_busy_end();-#endif- std::fill(h_part, h_part + (size_t) T * D, 0.0f);- for (int sl = 0; sl < nslots; ++sl) {- float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;- const float wv = cpu_rows[(size_t) sl].second;- const float* y = c_yd.data() + (size_t) sl * D;- for (int e = 0; e < D; ++e) o[e] += wv * y[e];- }- { static double ms = 0; static long calls = 0, exps = 0; static bool reg = false;- ms += now_ms() - tc; ++calls; exps += (long) ents.size();- if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu lane calls: %ld, experts %ld, %.1f ms (%.3f ms a call, %.3f ms an expert)%c", calls, exps, ms, ms / (double) calls, ms / (double) exps, 10); }); }- (void) 0; (void) ms; }- st.cpu_ms += now_ms() - tc;- dump_sub_host("cpu_part", l.il, p0, T, h_part, (size_t) T * D * 4);- CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));- }- combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);- dump_sub(d, "moe_out", l.il, p0, T, d.moe_out, (size_t) T * D * 4);- if (std::getenv("DS4_DBG_MOE")) {- float hp = 0.0f;- for (size_t i = 0; i < (size_t) T * D; ++i) hp = std::max(hp, std::fabs(h_part[i]));- std::fprintf(stderr, "DBG moe il=%d T=%d vram_rows=%d cpu_slots=%d max|h_part|=%.4g", l.il, T, nrows, nslots, (double) hp);- dbg_nf("moe_out", l.il, d.moe_out, (size_t) T * D);- dbg_nf("sh_out", l.il, d.sh_out, (size_t) T * D);- }- hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);- std::swap(d.h, d.h2);- dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);- if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, l.il, p0, T); // (post-layer residual, as the chunk path writes it)- CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));- if (phases && T == 1) {- const double ph4 = now_ms();- ph[0] += ph2 - ph0; ph[1] += ph3 - ph2; ph[2] += ph4 - ph3; if (ph_prev > 0 && ph0 - ph_prev < 5.0) ph[3] += ph0 - ph_prev; ph_prev = ph4; ++ph_n;- static bool reg = false;- if (!reg) { reg = true; std::atexit([] { std::fprintf(stderr, "phases per layer call (%ld): route enqueue + shared expert + wait %.3f ms, routing analysis + GPU experts enqueue %.3f ms, CPU lane + combine + hc_post %.3f ms, attention enqueue %.3f ms%s", ph_n, ph[0] / ph_n, ph[1] / ph_n, ph[2] / ph_n, ph[3] / ph_n, "\n"); }); }- }- // (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)- return true;- }-- int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter) {- Dev& d = devs[(size_t) head_dev];- cudaSetDevice(d.id);- strata::dsv41k::SamplerParams sp;- sp.temperature = temperature;- sp.greedy = temperature <= 0.0f;- sp.top_p = top_p;- sp.top_k = top_k;- sp.min_p = min_p;- sp.seed = seed;- sp.counter = counter;- strata::dsv41k::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,- sp, d.tok, d.st);- int t = -1;-#if defined(STRATA_METAL)- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)-#endif- cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);- cudaStreamSynchronize(d.st);- return t;- }-- bool logits(std::vector<float>& out) {- if (logits_all) {- out = chunk_logits;- return true;- }- Dev& d = devs[(size_t) head_dev];- cudaSetDevice(d.id);- out.resize((size_t) std::max(1, n_logits) * g.vocab);-#if defined(STRATA_METAL)- ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)-#endif- return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;- }-- void save_profile() const {- if (o.profile.empty()) return;- const std::string tmp = o.profile + ".tmp";- std::FILE* f = std::fopen(tmp.c_str(), "wb");- if (f == nullptr) return;- const bool ok = std::fwrite(prof.data(), 4, prof.size(), f) == prof.size();- std::fclose(f);- // (the old profile replaced: std::rename does not replace an existing file on Windows, so there the profile- // never moved on and only .tmp files were left)- if (!ok || !os::replace_file(tmp, o.profile)) std::remove(tmp.c_str());- }-- std::vector<std::string> info_lines() const {- std::vector<std::string> v;- int nh = 0;- for (const Layer& l : L) nh += l.n_exp - l.n_vram;- char b[512];- // kv=fp4: DeepSeek's own cache layout (the compressed KV and the index keys FP4, the window FP8)- std::snprintf(b, sizeof b, "INFO context=%lld kv=fp4 kv_resident=%lld expert_slots=%d engine_kind=ds4 "- "cpu_experts=%d ram_experts=%d engine=ds4-0.1",- (long long) o.max_context, (long long) o.max_context, g.nl * g.n_exp - nh, nh, ram_held);- v.push_back(b);- if (commit_kept > 0) { // (Windows, an integrated GPU: the commit kept experts off the GPU - the server says so)- std::snprintf(b, sizeof b,- "NOTE experts: %.0f GB stay off the GPU - Windows' page file is too small to hold them there; "- "a page file of %.0f GB keeps them on the GPU (System > About > Advanced system settings > "- "Performance > Advanced > Virtual memory), then restart Maya", commit_kept / 1e9, pagefile_hint);- v.push_back(b);- std::snprintf(b, sizeof b, "INFO gpu_commit_capped_gb=%.0f pagefile_hint_gb=%.0f", std::ceil(commit_kept / 1e9),- pagefile_hint);- v.push_back(b);- }- if (pk.eg_layers > 0) { // the n-gram memory's (Engram's) table rows a token reads, for the Monitor- std::snprintf(b, sizeof b, "INFO engram_layers=%d engram_lookups=%d", pk.eg_layers,- pk.eg_layers * (pk.eg_ngram - 1) * pk.eg_heads);- v.push_back(b);- }- if (img_tok >= 0) { // the server's image resize (DeepSeek's image_processor.py)- std::snprintf(b, sizeof b,- "INFO vision=ds4 vision_patch=%d vision_merge=%d vision_min_pixels=%lld vision_max_tokens=%d "- "vision_max_wh_ratio=%lld image_token=%d vision_gpu=%d vision_mib=%lld",- vg.patch, vg.merge, (long long) pk.num("vision.min_pixels", 0), vg.max_tokens,- (long long) pk.num("vision.max_wh_ratio", 0), img_tok, vt.device(),- (long long) (vt.device_bytes() >> 20));- v.push_back(b);- } else if (!vision_err.empty()) { // (the server offers no pictures and says why)- std::string why = vision_err;- for (char& ch : why)- if (ch == '\n' || ch == '\r') ch = ' ';- v.push_back("NOTE images: OFF - the model's vision tower did not load (" + why + ")");- v.push_back("INFO vision=off");- }- return v;- }-};--Model::Model() : p_(new Impl) {}-Model::~Model() = default;-bool Model::load(const Options& o, std::string& err) { return p_->load(o, err); }-bool Model::step(int token, bool logits, std::string& err) { return p_->step(token, logits, err); }-bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {- Impl& p = *p_;- p.pm.last = -1;- if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);- if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here- const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)- const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)- : p.feed_windows(tokens, n, logits_rows, err);- if (ok) p.path_done(now_ms() - t0);- return ok;- }- return p.feed_windows(tokens, n, logits_rows, err);-}-bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {- Impl& p = *p_;- p.pm.last = -1;- const double t0 = now_ms();- if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers- const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)- if (ok) p.path_done(now_ms() - t0);- return ok;- }- if (n > 1 && !p.feed_windows(tokens, n - 1, 0, err)) return false;- p.checkpoint(slot);- const bool ok = n < 1 || p.feed_windows(tokens + n - 1, 1, logits_rows, err);- if (ok) p.path_done(now_ms() - t0); // (windows, when pick_path chose them)- return ok;-}-int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }-void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }-int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {- return p_->sample(t, tp, tk, mp, seed, counter);-}-bool Model::logits(std::vector<float>& out) { return p_->logits(out); }-int Model::image_token() const { return p_->img_tok; }-int Model::dim() const { return p_->g.dim; }-bool Model::encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err) {- return p_->encode_image(rgb, width, height, span, err);-}-void Model::set_images(std::vector<Image> images) {- std::sort(images.begin(), images.end(), [](const Image& a, const Image& b) { return a.start < b.start; });- p_->imgs = std::move(images);-}-int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }-bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,- float mp, uint64_t seed, uint64_t counter) {- return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);-}-bool Model::verify(int y, const std::vector<int>& d, float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter,- int& accepted, int& next, std::string& err) {- return p_->verify(y, d, t, tp, tk, mp, seed, counter, accepted, next, err);-}-int64_t Model::pos() const { return p_->pos; }-int64_t Model::max_context() const { return p_->o.max_context; }-int Model::vocab() const { return p_->g.vocab; }-const std::vector<int32_t>& Model::tokens() const { return p_->toks; }-void Model::reset() { p_->reset(); }-void Model::checkpoint(int slot) { p_->checkpoint(slot); }-bool Model::restore(int64_t limit) { return p_->restore(limit); }-int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }-void Model::save_profile() const { p_->save_profile(); }-const Stats& Model::stats() const {- Stats& st = p_->st;- if (p_->store) { // (the store keeps its own counters: copied in when asked)- const ExpertStore& x = *p_->store;- st.store_hits = x.stats().hits.load();- st.store_misses = x.stats().misses.load();- st.store_prefetched = x.stats().prefetched.load();- st.store_pf_used = x.stats().prefetch_used.load();- st.store_pf_wasted = x.stats().prefetch_wasted.load();- st.store_evictions = x.stats().evictions.load();- st.store_bytes = x.stats().bytes_read.load();- st.store_fails = x.stats().fails.load();- st.store_wait_ms = x.stats().wait_ns.load() / 1e6;- st.store_busy_ms = x.busy_ms();- st.store_zc_visible = (int64_t) x.visible_count();- st.store_slots = (int64_t) x.slots_allocated();- st.store_slots_max = (int64_t) x.slots_max();- }- return st;-}-std::string Model::describe() const { return p_->describe(); }-std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }--} // namespace strata::ds4+// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).+// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.+// src/ds4/ds4_model.cu - DeepSeek-V4.1-Flash on Maya (see ds4_model.hpp).+//+// The layer, for one token at position p (DeepSeek's inference/model.py, Block.forward with the lagged pre-mix):+// [Engram, layers 1 and 14] h += gate(h, key) * value from 24 n-gram rows of the FP8 tables+// a_pre, a_post, a_comb = hc_mixes(h); x = rmsnorm(sum_s pre[s] h[s]) (pre: the previous layer's f_pre)+// attention(x): q from the low-rank wq_a / wq_b, one shared K = V row per token in a 128-slot window ring, plus+// (compress ratio > 0) up to 512 compressed rows the indexer picks; a learned sink per head; the+// output's rotary part rotated back, rounded to FP8, through the grouped wo_a and wo_b+// h = post * attn + comb^T h; f_* = hc_mixes(h); x = rmsnorm(sum_s a_pre[s] h[s])+// moe(x): 6 of 384 routed experts (sqrt-softplus scores, biased top-k) + the shared expert+// h = post * moe + comb^T h; pre = f_pre+// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal+#include "ds4_model.hpp"+#include "ds4_pick_rule.hpp"+#include "ds4_experts_rule.hpp"+#include "ds4_engram_io.hpp"++#include "ds4_gguf.cuh"+#include "ds4_iq.cuh"+#include "ds4_gguf_cpu.hpp"+#include "ds4_kernels.cuh"+#include "ds4_os.hpp"+#include "ds4_pool.hpp"+#include "ds4_store.hpp"+#include "ds4_vision.cuh"+#include "strata/dsv41/exl3_cpu.hpp"+#include "strata/dsv41/exl3.hpp"+#include "strata/kernels/f16_bits.hpp"+#include "strata/dsv41/moe_win.hpp"+#include "strata/dsv41/sampler.hpp"+#include "wmma_gemm.h" // strata_pf_gemm_f16_ld: the RDNA3 WMMA dense GEMM (gemm() prefers it over hipBLAS on gfx11; -Isrc/prefill)++#include <cublas_v2.h>+#include <cuda_runtime.h>+++#if defined(STRATA_METAL)++extern "C" void ds4_busy_begin();++extern "C" void ds4_busy_end();++#include <sys/sysctl.h>++#include <mach/mach.h>++#endif+#include <algorithm>+#include <array>+#include <condition_variable>+#include <deque>+#include <atomic>+#include <chrono>+#include <climits>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <filesystem>+#include <fstream>+#include <functional>+#include <future>+#include <map>+#include <mutex>+#include <numeric>+#include <sstream>+#include <thread>+#include <unordered_map>++#if defined(_MSC_VER)+#define strtok_r strtok_s+#endif++namespace strata::ds4 {+namespace x3 = strata::dsv41k::exl3;+namespace kc = strata::dsv41k::cpu;+namespace gf = strata::dsv41k::glmf;+namespace gg = strata::ds4::gguf;++#define CK(call) \+ do { \+ cudaError_t e_ = (call); \+ if (e_ != cudaSuccess) { \+ err = std::string(#call) + ": " + cudaGetErrorString(e_); \+ return false; \+ } \+ } while (0)++namespace {++// A HIP build (the share-based defaults below are gated on the devices being RDNA4, Impl::rdna4()).+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)+constexpr bool kHipBuild = true;+#else+constexpr bool kHipBuild = false;+#endif++// an environment variable as 0 / 1, or -1 when it is not set (the rules in ds4_experts_rule.hpp take this form)+int env_tri(const char* name) {+ const char* v = std::getenv(name);+ return v != nullptr ? (std::atoi(v) != 0 ? 1 : 0) : -1;+}++double now_ms() {+ return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();+}++size_t align256(size_t n) { return (n + 255) & ~(size_t) 255; }++// part p of np of `whole` (a multiple of 128) in 128-blocks, as even as they go (whole / 128 % np parts a block more:+// those from part rot on, cyclically - so a group's layers, each its own rot, even out what each member holds):+// tensor parallel's columns of an FFN or of the head - an even split is what whole / np gave before+void split128(int whole, int np, int p, int& off, int& n, int rot = 0) {+ const int b = whole / 128, q = b / np, r = b % np;+ auto blocks = [&](int i) { return q + (((i - rot) % np + np) % np < r ? 1 : 0); };+ int at = 0;+ for (int i = 0; i < p; ++i) at += blocks(i);+ off = 128 * at;+ n = 128 * blocks(p);+}++std::string read_file(const std::string& p) {+ std::ifstream f(p);+ std::stringstream ss;+ ss << f.rdbuf();+ return ss.str();+}+std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"+ std::vector<int> v;+ std::stringstream ss(s);+ std::string part;+ while (std::getline(ss, part, ',')) {+ if (part.empty() || part == "\n") continue;+ const size_t dash = part.find('-');+ const int a = std::atoi(part.c_str());+ const int b = dash == std::string::npos ? a : std::atoi(part.c_str() + dash + 1);+ for (int c = a; c <= b; ++c) v.push_back(c);+ }+ return v;+}+// each NUMA node's physical cores (the first logical CPU of each core), in order+std::vector<std::vector<int>> node_cores() {+#if defined(_WIN32)+ static const bool smt = [] { // DS4_CPU_SMT=1: the second hardware thread of every core as well (memory-bound rows gain from it)+ const char* v = std::getenv("DS4_CPU_SMT");+ return v != nullptr && std::atoi(v) != 0;+ }();+ std::vector<std::vector<int>> nodes = os::node_cores(smt);+#else+ std::vector<std::vector<int>> nodes;+ for (int n = 0; n < 64; ++n) {+ const std::string cl = read_file("/sys/devices/system/node/node" + std::to_string(n) + "/cpulist");+ if (cl.empty()) break;+ std::vector<int> cores;+ for (int c : cpu_list(cl)) {+ const std::vector<int> sib = cpu_list(+ read_file("/sys/devices/system/cpu/cpu" + std::to_string(c) + "/topology/thread_siblings_list"));+ if (sib.empty() || sib[0] == c) cores.push_back(c);+ }+ if (!cores.empty()) nodes.push_back(cores);+ }+#endif+ if (nodes.empty()) {+ std::vector<int> all;+ for (unsigned c = 0; c < std::max(1u, std::thread::hardware_concurrency() / 2); ++c) all.push_back((int) c);+ nodes.push_back(all);+ }+ return nodes;+}+// anonymous memory on one NUMA node (node < 0: wherever the first touch puts it), transparent huge pages+void* node_map(size_t bytes, int node, int n_nodes) { return os::anon_alloc(bytes, node, n_nodes); }+void pin_self(int cpu) { os::pin_self(cpu); }+// the RAM this process can still get: MemAvailable, and below it the tightest cgroup v2 limit on the way to the root+// (its limit less what is charged, the page cache counted as free: the kernel takes it back); ~0: not known+uint64_t ram_avail() {+#if defined(_WIN32)+ return os::ram_avail();+#endif+ uint64_t avail = ~uint64_t{0};+ {+ std::istringstream mi(read_file("/proc/meminfo"));+ std::string k;+ uint64_t kb = 0;+ while (mi >> k >> kb) {+ if (k == "MemAvailable:") avail = kb << 10;+ mi.ignore(64, '\n');+ }+ }+ std::string cg = read_file("/proc/self/cgroup"); // "0::/user.slice/..."+ const size_t at = cg.find("0::");+ if (at == std::string::npos) return avail;+ cg = cg.substr(at + 3);+ cg = cg.substr(0, cg.find('\n'));+ for (;;) {+ const std::string dir = "/sys/fs/cgroup" + (cg == "/" ? std::string() : cg);+ const std::string mx = read_file(dir + "/memory.max");+ if (!mx.empty() && mx.compare(0, 3, "max") != 0) {+ const uint64_t lim = std::strtoull(mx.c_str(), nullptr, 10);+ const uint64_t cur = std::strtoull(read_file(dir + "/memory.current").c_str(), nullptr, 10);+ uint64_t file = 0;+ std::istringstream st(read_file(dir + "/memory.stat"));+ std::string k;+ uint64_t v = 0;+ while (st >> k >> v)+ if (k == "file") file = v;+ const uint64_t used = cur > file ? cur - file : 0;+ avail = std::min(avail, lim > used ? lim - used : 0);+ }+ if (cg.empty() || cg == "/") break;+ const size_t sl = cg.rfind('/');+ cg = sl == 0 || sl == std::string::npos ? "/" : cg.substr(0, sl);+ }+ return avail;+}++float bf16_to_f32(uint16_t b) {+ uint32_t u = (uint32_t) b << 16;+ float f;+ std::memcpy(&f, &u, 4);+ return f;+}+float f16_to_f32(uint16_t h) {+ const uint32_t s = (uint32_t) (h >> 15) << 31, e = (h >> 10) & 0x1f, m = h & 0x3ff;+ uint32_t u;+ if (e == 0) {+ if (m == 0) u = s;+ else { // subnormal+ int ee = -1;+ uint32_t mm = m;+ do {+ ++ee;+ mm <<= 1;+ } while ((mm & 0x400) == 0);+ u = s | (uint32_t) (127 - 15 - ee) << 23 | (mm & 0x3ff) << 13;+ }+ } else if (e == 31) u = s | 0x7f800000u | m << 13;+ else u = s | (e - 15 + 127) << 23 | m << 13;+ float f;+ std::memcpy(&f, &u, 4);+ return f;+}++// ------------------------------------------------------------------ the pack+struct Tensor {+ std::string dtype;+ int file = -1;+ uint64_t off = 0, bytes = 0;+ std::vector<int64_t> shape;+};++struct Pack {+ std::string dir, model_dir;+ std::unordered_map<std::string, std::string> kv;+ std::vector<std::string> files;+ std::vector<os::fd_t> fds;+ std::unordered_map<std::string, Tensor> t;+ int eg_vocab = 0, eg_layers = 0, eg_ngram = 0, eg_heads = 0, eg_hd = 0, eg_pad = 0;+ std::vector<int32_t> eg_map;+ std::vector<int64_t> eg_mult, eg_primes, eg_off;+ // format gguf (tools/ds4/pack_gguf.py): the files are a GGUF's shards, every one mapped at load (map()) - the+ // routed experts are always read in place (the OS page cache their RAM tier, the disk the rest)+ bool gguf = false;++ std::vector<std::pair<uint8_t*, size_t>> maps; // file -> its read-only mapping (map(): the experts in place)+ std::vector<os::fd_t> dfds; // (gguf) file -> unbuffered (os::open_direct; kBadFd: buffered reads only)+ std::vector<os::fd_t> pfds; // (gguf) file -> for reads from many threads at once (os::open_par)++ ~Pack() {+ for (auto& m : maps) os::unmap(m.first, m.second);+ for (os::fd_t fd : fds) os::close_fd(fd);+ for (os::fd_t fd : dfds) os::close_fd(fd);+ for (os::fd_t fd : pfds) os::close_fd(fd);+ }+ // file i mapped whole, read-only (once; not thread-safe: called while loading)+ const uint8_t* map(int file) {+ if (maps.size() < fds.size()) maps.resize(fds.size(), {nullptr, 0});+ auto& m = maps[(size_t) file];+ if (m.first == nullptr) {+ const uint64_t size = os::file_size(fds[(size_t) file]);+ void* p = os::map_ro(fds[(size_t) file], size);+ if (p == nullptr) return nullptr;+ m = {(uint8_t*) p, (size_t) size};+ }+ return m.first;+ }++ bool load(const std::string& d, std::string& err) {+ dir = d;+ {+ std::ifstream f(d + "/ds4.txt");+ if (!f) {+ err = d + "/ds4.txt: not found (run tools/ds4/pack.py --model <model>)";+ return false;+ }+ std::string line;+ while (std::getline(f, line)) {+ const size_t sp = line.find(' ');+ if (sp == std::string::npos) continue;+ kv[line.substr(0, sp)] = line.substr(sp + 1);+ }+ }+ model_dir = str("model_dir");+ std::FILE* f = std::fopen((d + "/tensors.txt").c_str(), "r");+ if (f == nullptr) {+ err = d + "/tensors.txt: not found";+ return false;+ }+ std::vector<char> buf(1 << 16);+ while (std::fgets(buf.data(), (int) buf.size(), f) != nullptr) {+ char* s = buf.data();+ if (s[0] == 'f') { // file <i> <name>+ char name[4096];+ int i = 0;+ if (std::sscanf(s, "file %d %4095s", &i, name) == 2) {+ if ((int) files.size() <= i) files.resize((size_t) i + 1);+ files[(size_t) i] = name;+ }+ continue;+ }+ if (s[0] != 't') continue;+ char* save = nullptr;+ ::strtok_r(s, " \n", &save);+ const char* name = ::strtok_r(nullptr, " \n", &save);+ const char* dt = ::strtok_r(nullptr, " \n", &save);+ const char* fi = ::strtok_r(nullptr, " \n", &save);+ const char* off = ::strtok_r(nullptr, " \n", &save);+ const char* nb = ::strtok_r(nullptr, " \n", &save);+ if (name == nullptr || nb == nullptr) continue;+ Tensor x;+ x.dtype = dt;+ x.file = std::atoi(fi);+ x.off = std::strtoull(off, nullptr, 10);+ x.bytes = std::strtoull(nb, nullptr, 10);+ while (const char* dim = ::strtok_r(nullptr, " \n", &save)) x.shape.push_back(std::atoll(dim));+ t.emplace(name, std::move(x));+ }+ std::fclose(f);+ for (const std::string& fn : files) {+ const std::string p = model_dir + "/" + fn;+ const os::fd_t fd = os::open_ro(p);+ if (fd == os::kBadFd) {+ err = p + ": cannot open";+ return false;+ }+ fds.push_back(fd);+ }+ gguf = str("format") == "gguf";+ if (gguf)+ for (size_t i = 0; i < fds.size(); ++i) {+ if (map((int) i) == nullptr) {+ err = model_dir + "/" + files[i] + ": cannot map";+ return false;+ }+ dfds.push_back(os::open_direct(model_dir + "/" + files[i]));+ pfds.push_back(os::open_par(model_dir + "/" + files[i]));+ }+ // the Engram tables+ std::FILE* e = std::fopen((d + "/engram.bin").c_str(), "rb");+ if (e == nullptr) {+ err = d + "/engram.bin: not found";+ return false;+ }+ char magic[4];+ int32_t hdr[6];+ bool ok = std::fread(magic, 1, 4, e) == 4 && std::memcmp(magic, "DSE1", 4) == 0 &&+ std::fread(hdr, 4, 6, e) == 6;+ if (ok) {+ eg_vocab = hdr[0], eg_layers = hdr[1], eg_ngram = hdr[2], eg_heads = hdr[3], eg_hd = hdr[4], eg_pad = hdr[5];+ const int cols = (eg_ngram - 1) * eg_heads;+ eg_map.resize((size_t) eg_vocab);+ eg_mult.resize((size_t) eg_layers * eg_ngram);+ eg_primes.resize((size_t) eg_layers * cols);+ eg_off.resize((size_t) eg_layers * cols);+ ok = std::fread(eg_map.data(), 4, eg_map.size(), e) == eg_map.size() &&+ std::fread(eg_mult.data(), 8, eg_mult.size(), e) == eg_mult.size() &&+ std::fread(eg_primes.data(), 8, eg_primes.size(), e) == eg_primes.size() &&+ std::fread(eg_off.data(), 8, eg_off.size(), e) == eg_off.size();+ }+ std::fclose(e);+ if (!ok) {+ err = d + "/engram.bin: unreadable";+ return false;+ }+ return true;+ }++ std::string str(const std::string& k, const std::string& def = "") const {+ auto it = kv.find(k);+ return it == kv.end() ? def : it->second;+ }+ double num(const std::string& k, double def) const {+ auto it = kv.find(k);+ return it == kv.end() ? def : std::atof(it->second.c_str());+ }+ std::vector<int> list(const std::string& k) const {+ std::vector<int> v;+ std::stringstream ss(str(k));+ std::string x;+ while (std::getline(ss, x, ','))+ if (!x.empty()) v.push_back(std::atoi(x.c_str()));+ return v;+ }+ const Tensor* get(const std::string& n) const {+ auto it = t.find(n);+ return it == t.end() ? nullptr : &it->second;+ }+ bool read_at(int file, uint64_t off, uint64_t n, void* dst) const { // (any thread; side by side where pfds is)+ if ((size_t) file < pfds.size() && pfds[(size_t) file] != os::kBadFd)+ return os::pread_par(pfds[(size_t) file], dst, n, off);+ return os::pread_full(fds[(size_t) file], dst, n, off);+ }+ bool read(const Tensor& x, void* dst) const { return read_at(x.file, x.off, x.bytes, dst); }+ const uint8_t* at(int file, uint64_t off) const { return maps[(size_t) file].first + off; } // (gguf: mapped)+};++// ------------------------------------------------------------------ geometry+struct Geo {+ int vocab = 0, dim = 0, nl = 0, heads = 0, hd = 0, rd = 0, q_lora = 0, o_lora = 0, o_groups = 0, n_exp = 0,+ k_exp = 0, ff = 0, win = 0, hc = 0, sinkhorn = 0, ix_heads = 0, ix_hd = 0, ix_topk = 0, cand_src = -1,+ cand_topk = 0, cand_blk = 0, orig_ctx = 0, eos = 1;+ float eps = 1e-20f, hc_eps = 1e-6f, route_scale = 1.0f, lim = 0.0f;+ double rope_theta = 1e4, comp_theta = 1e4, factor = 1.0, beta_fast = 32, beta_slow = 1;+ std::vector<int> ratios, kv_src, ix_src, eg_layers;+ // the DSpark drafter (the checkpoint's mtp.* blocks): a block of [seed, noise x (block - 1)] through n_mtp blocks+ int n_mtp = 0, d_block = 0, d_noise = 0, d_rank = 0, d_n_exp = 0, d_k = 0;+ std::vector<int> d_targets;++ void from(const Pack& p) {+ vocab = (int) p.num("vocab_size", 0), dim = (int) p.num("hidden_size", 0);+ nl = (int) p.num("num_hidden_layers", 0), heads = (int) p.num("num_attention_heads", 0);+ hd = (int) p.num("head_dim", 0), rd = (int) p.num("qk_rope_head_dim", 0);+ q_lora = (int) p.num("q_lora_rank", 0), o_lora = (int) p.num("o_lora_rank", 0);+ o_groups = (int) p.num("o_groups", 0), n_exp = (int) p.num("n_routed_experts", 0);+ k_exp = (int) p.num("num_experts_per_tok", 0), ff = (int) p.num("moe_intermediate_size", 0);+ win = (int) p.num("sliding_window", 0), hc = (int) p.num("hc_mult", 0);+ sinkhorn = (int) p.num("hc_sinkhorn_iters", 0), ix_heads = (int) p.num("index_n_heads", 0);+ ix_hd = (int) p.num("index_head_dim", 0), ix_topk = (int) p.num("index_topk", 0);+ cand_src = (int) p.num("candidate_source_layer_id", -1), cand_topk = (int) p.num("candidate_topk_blocks", 0);+ cand_blk = (int) p.num("candidate_block_size", 0);+ orig_ctx = (int) p.num("rope_scaling.original_max_position_embeddings", 0);+ eos = (int) p.num("eos_token_id", 1);+ eps = (float) p.num("rms_norm_eps", 1e-20), hc_eps = (float) p.num("hc_eps", 1e-6);+ route_scale = (float) p.num("routed_scaling_factor", 1.0), lim = (float) p.num("swiglu_limit", 0.0);+ rope_theta = p.num("rope_theta", 1e4), comp_theta = p.num("compress_rope_theta", 1e4);+ factor = p.num("rope_scaling.factor", 1.0), beta_fast = p.num("rope_scaling.beta_fast", 32);+ beta_slow = p.num("rope_scaling.beta_slow", 1);+ ratios = p.list("compress_ratios"), kv_src = p.list("kv_source_layer_ids");+ ix_src = p.list("index_source_layer_ids"), eg_layers = p.list("engram_layer_ids");+ d_block = (int) p.num("dspark_block_size", 0), d_noise = (int) p.num("dspark_noise_token_id", 0);+ d_rank = (int) p.num("dspark_markov_rank", 0), d_n_exp = (int) p.num("dspark_n_routed_experts", 0);+ d_k = (int) p.num("dspark_num_experts_per_tok", 0), d_targets = p.list("dspark_target_layer_ids");+ // the DSpark blocks: an EXL3 pack's, or a GGUF pack's from DeepSeek's checkpoint (tools/ds4/drafter_pack.py)+ n_mtp = d_block > 0 && (int) ratios.size() > nl &&+ (p.get("mtp.0.attn.wq_a.trellis") != nullptr || p.get("mtp.0.attn.wq_a.weight") != nullptr)+ ? (int) ratios.size() - nl : 0;+ }+};++// DeepSeek's precompute_freqs_cis frequencies (YaRN when original_seq_len > 0), in F32 as torch computes them+std::vector<float> rope_freqs(int dim, int original, double base, double factor, double beta_fast, double beta_slow) {+ std::vector<float> f((size_t) dim / 2);+ for (int i = 0; i < dim / 2; ++i) {+ const float e = (float) (2 * i) / (float) dim;+ f[(size_t) i] = 1.0f / std::pow((float) base, e);+ }+ if (original > 0) {+ auto corr = [&](double rot) {+ return dim * std::log(original / (rot * 2 * 3.14159265358979323846)) / (2 * std::log(base)); // (M_PI: not MSVC's)+ };+ const double low = std::max(std::floor(corr(beta_fast)), 0.0);+ const double high = std::min(std::ceil(corr(beta_slow)), (double) dim - 1);+ const float den = (float) std::max(high - low, 1e-3);+ for (int i = 0; i < dim / 2; ++i) {+ float ramp = ((float) i - (float) low) / den;+ ramp = std::min(1.0f, std::max(0.0f, ramp));+ const float smooth = 1.0f - ramp;+ f[(size_t) i] = f[(size_t) i] / (float) factor * (1.0f - smooth) + f[(size_t) i] * smooth;+ }+ }+ return f;+}++// ------------------------------------------------------------------ devices, layers+struct Dev {+ int id = 0;+ cudaStream_t st = nullptr;- cudaEvent_t ev = nullptr;++ cudaEvent_t ev = nullptr, ev2 = nullptr; // (ev2: the lookahead's guesses are on the host)++ int* ridx_a = nullptr; // (Metal) the next layers' guessed picks (2 x kPicks)++ float* rw_a = nullptr;+ size_t free0 = 0, total = 0, weights = 0, experts = 0, caches = 0, scratch = 0;+ std::vector<void*> allocs;+ // a window's scratch, rows of up to kWin tokens (the routed experts' rows: kWin * kPicks)+ float *h = nullptr, *h2 = nullptr, *x = nullptr, *qr = nullptr, *q = nullptr, *kvr = nullptr, *ckr = nullptr,+ *cgr = nullptr, *latent = nullptr, *ikr = nullptr, *iq = nullptr, *iw = nullptr, *scores = nullptr,+ *bm = nullptr, *o = nullptr, *oa = nullptr, *ao = nullptr, *mix = nullptr, *a_pre = nullptr,+ *a_post = nullptr, *a_comb = nullptr, *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr,+ *glog = nullptr, *rw = nullptr, *moe_out = nullptr, *sg = nullptr, *su = nullptr, *sh = nullptr,+ *sh_out = nullptr, *cpu_part = nullptr, *eg_x = nullptr, *eg_kv = nullptr, *logits = nullptr,+ *freqs_win = nullptr, *freqs_comp = nullptr, *xe = nullptr, *xg = nullptr, *xu = nullptr, *xhh = nullptr,+ *ed = nullptr;+ float* glog_a = nullptr; // lookahead: the next two layers' gate logits on this layer's FFN input (rows 1024 apart)+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *tok = nullptr;+ unsigned long long* plan_ptr = nullptr; // the decode path's VRAM experts (0: on the CPU lane)+ uint8_t* eg_rows = nullptr;+ uint16_t *emb_rows = nullptr, *ext = nullptr; // the window's embedding rows; the window attention's rows+ AttnScratch attn_scr; // a few queries' selected compressed rows decoded (sparse_attn)+ // the hand-off to the next device through pinned host memory (these cards copy peer to peer only through the+ // driver's staging, ~1.7 ms a copy): h | pre | topk | cand+ uint8_t* bounce = nullptr;+ cudaEvent_t ev_out = nullptr;+ float* tap = nullptr; // a window's stream means before a DSpark target layer+ // threads / tensor parallelism: its group and its index in it, the group's all-reduces it took part in, xfer_put's+ // block counter, the hand-off to the next group (host-mapped: h | pre | topk | cand, then its flag), the window+ // MoE's grouping scratch+ int grp = 0, mi = 0;+ unsigned ar_seq = 0;+ unsigned* xcnt = nullptr;+ unsigned* hccnt = nullptr; // [kWin] hc_mix_split's per-token block counters (zeroed at allocation, left zero)+ // the head group's members (the last layer's group): each its columns (split128 of V) of the head and the norms;+ // logits_part: a member's share of the logits (the lead's: where the others' land before they join its rows)+ x3::Mat head;+ float *norm_w = nullptr, *d_norm = nullptr, *logits_part = nullptr;+ unsigned head_seq = 0;+ // the drafter's markov bias over the head group (markov_share): this member's rows [mk_v0, mk_v0 + mk_vn) of the+ // markov head, the previous token's E row, the drafted tokens (the lead's: d_ints)+ uint16_t *mk_share = nullptr, *mk_erow = nullptr;+ int* mk_ints = nullptr;+ int mk_v0 = 0, mk_vn = 0;+ uint8_t* hand = nullptr;+ unsigned* hand_flag = nullptr;+ unsigned long long* grp_ptr = nullptr;+ int *grp_start = nullptr, *grp_ent = nullptr, *n_groups = nullptr;+};++// the chunked prompt path's buffers on a device: a chunk of C tokens through a layer at once (the attention in slabs of+// A queries, the indexer in slabs of AI), its GEMMs through FP16 reconstructions of the weights and cuBLAS+// DS4_PRED_DUMP: a fixed +-1 random projection of the router input (D -> 256), one block a token+__global__ void pred_proj_k(const float* x, int D, float* out) {+ const int t = blockIdx.x, j = threadIdx.x; // 256 threads+ const float* xr = x + (size_t) t * D;+ float a = 0.0f;+ for (int d = 0; d < D; ++d) {+ unsigned h = (unsigned) d * 2654435761u ^ ((unsigned) j + 1u) * 40503u;+ h ^= h >> 15; h *= 2246822519u; h ^= h >> 13; h *= 3266489917u; h ^= h >> 16;+ a += (h & 1u) ? xr[d] : -xr[d];+ }+ out[(size_t) t * 256 + j] = a * rsqrtf((float) D);+}++struct Chunk {+ int C = 0, A = 0, AI = 0;+ float *h = nullptr, *x = nullptr, *acc = nullptr, *qr = nullptr, *kvr = nullptr, *ckr = nullptr, *cgr = nullptr,+ *latent = nullptr, *ikr = nullptr, *q = nullptr, *o = nullptr, *oa = nullptr, *iq = nullptr, *iw = nullptr,+ *scores = nullptr, *bm = nullptr, *mix = nullptr, *a_pre = nullptr, *a_post = nullptr, *a_comb = nullptr,+ *f_pre = nullptr, *f_post = nullptr, *f_comb = nullptr, *pre = nullptr, *glog = nullptr, *rw = nullptr,+ *sg = nullptr, *su = nullptr, *y = nullptr, *eg_x = nullptr, *eg_kv = nullptr;+ uint16_t *x16 = nullptr, *qr16 = nullptr, *o16 = nullptr, *oa16 = nullptr, *lat16 = nullptr, *sh16 = nullptr,+ *h16 = nullptr, *eg_x16 = nullptr, *ext = nullptr, *w16 = nullptr, *emb = nullptr;+ size_t w16_elems = 0;+ int *ridx = nullptr, *topk = nullptr, *cand = nullptr, *bounds = nullptr, *row_tok = nullptr, *pos = nullptr;+ uint8_t* eg_rows = nullptr;+ int* bounds_pin = nullptr; // pinned: the sort's bounds, read back+ // the CPU lane's experts streamed in: ring_n VRAM slots of ring_stride bytes in kParts parts, a batch a part,+ // kParts batches in flight on the copy stream (filled / freed: each part's last copy, its last use)+ static constexpr int kParts = 4;+ uint8_t* ring = nullptr;+ size_t ring_stride = 0;+ int ring_n = 0;+ cudaStream_t cs = nullptr;+ cudaEvent_t filled[kParts] = {}, freed[kParts] = {};+ // in place (file_experts): each streamed expert's file region read into pinned staging (ring_n slots of+ // stage_stride bytes, a part's slots free again once its `filled` copies are done) by the readers, in parallel,+ // and DMA'd from there - a copy from the files' pages goes through the driver's staging, one thread at ~1.3 GB/s+ uint8_t* stage = nullptr;+ size_t stage_stride = 0;+ std::unique_ptr<Pool> readers;+ cublasHandle_t blas = nullptr;+ // host-mapped: a chunk's hand-off to the next group (h | pre | topk | cand), two in flight (slot = chunk % 2), each+ // with its flag, and the next group's members' acknowledgements [slot][member] (32 apart)+ uint8_t* hand[2] = {nullptr, nullptr};+ unsigned* hand_flag[2] = {nullptr, nullptr};+ unsigned* hand_ack = nullptr;+ unsigned arb_seq = 0; // the group's chunk all-reduces+ // pinned, two chunks in flight: the embedding rows (the first layer's device), the Engram rows (an Engram layer's)+ uint16_t* emb_host[2] = {nullptr, nullptr};+ uint8_t* eg_host[2] = {nullptr, nullptr};+ cudaEvent_t emb_ev[2] = {nullptr, nullptr}, eg_ev[2] = {nullptr, nullptr};+ double stream_bytes = 0; // the experts streamed in, in all+ // the flow (a GGUF's in-place experts, every layer of the device on it alone; DS4_FLOW=0: the batches above): a+ // (chunk, layer)'s streamed experts are those it routes to - its likely ones (the profile's chance of a pick by the+ // chunk's tokens) queued once the layer before it routed, read while that one computes, the rest of its picks once+ // it routed itself; never an expert no token of the chunk picked unless it was a likely one. Each item read by io+ // (unbuffered: the disk straight into its staging slot, every part of every item in flight at once - none waits+ // for a batch) or taken from its RAM-tier slot as it lies, then DMA'd into the ring in order, item i into slot+ // i % ring_n; the compute takes the items as they land, in runs of what has.+ struct Item {+ int il = 0, k = 0, e = 0, v = 0, stage = -1; // layer, chunk, expert, virtual slot, staging slot (-1: RAM)+ int mi = 0; // the device's member index in the layer's group++ bool direct = false; // (Metal) read straight into its ring slot: no staging, no DMA+ const uint8_t* src[3] = {nullptr, nullptr, nullptr}; // where its gate / up / down (its parts) lie once read+ const uint8_t* ram = nullptr; // its RAM-tier blob, copied into staging (not pinned there)+ uint8_t* keep = nullptr; // (layer-major) its layer-cache slot: what is read kept there+ int64_t after = -1; // (layer-major) the items up to this one read first+ std::atomic<int> left{0}; // parts still being read+ std::atomic<bool> bad{false};+ };+ bool flow = false;++ bool direct = false; // (Metal, unified memory) the flow reads the files straight into the ring+ int dev = -1; // the device (index into devs) this chunk state belongs to+ std::unique_ptr<IoQueue> io;+ std::deque<Item> items; // items[0] is item `base`; the indices below are absolute+ size_t base = 0, n_read = 0, n_dma = 0, n_comp = 0;+ uint64_t n_file = 0; // file items so far (staging slot = ordinal % S)+ int S = 0; // staging slots, stage_stride apart+ std::vector<int64_t> stage_item; // staging slot -> the item last read into it (-1: none)+ std::vector<cudaEvent_t> stage_ev, ring_fill, ring_free; // a staging slot's DMA out; a ring slot's DMA in, its use+ int spec_k = -1, spec_il = -1, n_spec = 0; // the likely items queued last: (chunk, layer), how many, from where+ size_t spec_begin = 0;+ std::vector<int> fl_layers, fl_T; // the device's trunk layers in order; the run's chunk sizes+ int fl_yoco = -1, fl_cut_k = 0, fl_cut_r = 0; // Run::yoco / cut_k / cut_r: what a layer's chunk streams for+ int bc_n = 0, bc_w = 0; // a batch's compacted bounds: bc_n regions of bc_w, by batch count+ int *bc_pin = nullptr, *bc_dev = nullptr;+ uint64_t n_batch = 0;+ double wait_ms = 0; // the compute waiting for a read+ // (DS4_PROF: where copy_ahead's time goes - the host staging copy, the DMA issues, the waits for a slot)+ double dbg_stage_ms = 0, dbg_dma_ms = 0, dbg_evt_ms = 0;+ uint64_t it_file = 0, it_ram = 0, it_spec = 0, it_spec_unused = 0;+ uint64_t io_d0 = 0, io_b0 = 0; // io's unbuffered / buffered bytes when the run started+ double io_busy0 = 0; // io's busy time when the run started (DS4_PROF)+ // layer-major (a run of several chunks, every layer on this device): the layers one after another, each through+ // all the chunks; a chunk's state between its layers (h | pre | topk | cand, chunk k at k chunk_hand_bytes()) in+ // host memory, and the layer cache - RAM slots where the experts a layer's chunk reads from the files are kept for+ // its later chunks, so each is read once a run+ bool lm = false, lm_pinned = false;+ uint8_t* lm_state = nullptr;+ size_t lm_state_bytes = 0;+ uint8_t* lc = nullptr; // lc_bytes of slots, a layer's experts' blobs side by side+ size_t lc_bytes = 0;+ int lc_il = -1, lc_used = 0, lc_n = 0; // the layer the cache holds, its slots taken, its slots in all+ size_t lc_first = 0, lc_chk = 0; // the layer's first item; the items before lc_chk all read+ std::vector<int> lc_of; // expert -> its slot (-1: none)+ std::vector<int64_t> lc_writer; // slot -> the item reading its expert into it+ uint64_t it_lc = 0; // items taken from the layer cache+};++struct Layer {+ int il = 0, dev = 0, ratio = 0;+ // its group (tensor parallel: nm members, each a copy of the device state in mates, member mi's share of the+ // heads, of wo_a's groups, of wo_b's rows and of the FFNs: gate / up columns, down rows; its part of an expert+ // laid out as hlay, hblob bytes - the host state, the CPU lane and the full blob layout stay with member 0)+ int grp = 0, nm = 1, mi = 0;+ std::vector<Layer> mates;+ x3::ExpertLayout hlay;+ size_t hblob = 0;+ int ff0 = 0, ffn = 0; // this member's FFN columns [ff0, ff0 + ffn) (split128 of n_ff; the whole one alone)+ // this member's heads [ah0, ah0 + ahn) and wo_a groups [og0, og0 + ogn) - all of them alone; in a group of more+ // members than the heads split among evenly (head_members: 9, 7 or 6 over 64 heads), the first ones none: their+ // attention adds nothing to the sum+ int ah0 = 0, ahn = 0, og0 = 0, ogn = 0;+ // the chunked prompt path: each expert's virtual slot (its VRAM slot, or n_slots + its place among the CPU lane's+ // experts, which a chunk streams in that order) - pinned on member 0, a device copy on each member+ int* vs_pin = nullptr;+ int* vs_dev = nullptr;+ std::vector<int> host_list;+ std::string name; // "layers.N." or, for the DSpark drafter's blocks, "mtp.N."+ int n_exp = 0, k_exp = 0; // routed experts, and how many a token takes (the drafter's: 128, 3)+ bool mtp = false; // a DSpark block (index stage = il - n_layers)+ bool kv_src = false, ix_src = false, owns_k = false, cand_src = false, uses_cand = false;+ int src = -1; // the kv source whose compressed KV this layer reads (index into Impl::srcs)+ int ik_src = -1; // the kv source whose index keys its indexer reads+ int eg = -1; // engram layer index, or -1+ x3::Mat wq_a, wq_b, wkv, wo_a[16], wo_b, s1, s2, s3, c_wkv, c_wgate, i_wq_b, i_wk, eg_wkv;+ float *q_norm = nullptr, *kv_norm = nullptr, *attn_norm = nullptr, *ffn_norm = nullptr, *c_norm = nullptr,+ *ik_norm = nullptr, *sink = nullptr, *hca_fn = nullptr, *hca_scale = nullptr, *hca_base = nullptr,+ *hcf_fn = nullptr, *hcf_scale = nullptr, *hcf_base = nullptr, *gate_bias = nullptr, *eg_w = nullptr,+ *gate_bias_vl = nullptr; // (the image tokens' routing bias, with the vision tower)+ uint16_t* i_wproj = nullptr;+ float* gate_w = nullptr; // the router's weights, F32 (DeepSeek's inference computes the gate in F32)+ uint8_t* win = nullptr; // [win] rows of kKvFp8 (kv_row_bytes)+ float *c_kv = nullptr, *c_score = nullptr; // the compressor's partial group (ratio > 1)+ // a run of chunks' mid-run checkpoint (feed_mark): the ring and the partial group as they were before a row+ uint8_t* snap_win = nullptr;+ float *snap_kv = nullptr, *snap_sc = nullptr;+ // routed experts+ x3::ExpertLayout lay;+ size_t blob = 0; // bytes of one expert's blob+ int n_vram = 0;+ bool cpu_only = false; // a role's type has no device kernel: every expert of this layer is the CPU lane's+ uint8_t* vram_base = nullptr;+ std::vector<int> slot; // expert -> VRAM slot or -1+ std::vector<int> ram; // expert -> RAM-tier slot or -1 (a GGUF's in-place experts; empty: none)+ std::vector<uint8_t*> host; // expert -> host (non-null: on the CPU lane; its halves in hh)+ // the CPU lane keeps an expert as NH column parts, part X on NUMA node X: gate / up columns [X n_ff / NH, ..),+ // down columns [X dim / NH, ..), each a standalone EXL3 matrix (its own trellis, the full suh, its part of svh)+ struct Half {+ size_t trellis[3], suh[3], svh[3], bytes;+ int k[3], n[3];+ } hl;+ std::vector<std::array<uint8_t*, 4>> hh; // expert -> its parts on the nodes+ // the expert cache: one VRAM slot and one set of host parts kept free, so an expert can move in while the one it+ // replaces moves out; heat is how much each expert was used lately (the profile's share to start with)+ int n_slots = 0, spare_vram = -1;+ std::array<uint8_t*, 4> spare_host = {nullptr, nullptr, nullptr, nullptr};+ std::vector<float> heat;+ std::vector<int> cpu_hits; // CPU-lane experts used since the cache last looked+ bool swap_pending = false;+ unsigned long long* tab_pin = nullptr; // pinned: the table entries a swap writes+ // a speculative window's backups (trunk layers): the ring rows it replaced, and a compressor's state before it and+ // the window's raw rows (kv, then score) to replay the accepted ones+ uint8_t* ring_bak = nullptr;+ float *c_snap = nullptr, *c_rows = nullptr;+ unsigned long long* vram_tab = nullptr; // device [n_exp]+ std::vector<uint8_t> zc; // (the RAM store) expert -> 1: its store slot is GPU-visible, the device computes it over PCIe+ unsigned long long* zc_tab = nullptr; // device [n_exp]: those slots' device addresses (0: none) - the decode token's zc_stage reads it+ std::vector<std::array<x3::Mat, 3>> vm; // expert -> its gate, up, down as EXL3 views of its VRAM blob+ std::vector<uint64_t> region_off; // expert -> file region (offset, bytes, file)+ std::vector<uint64_t> region_bytes;+ std::vector<int> region_file;+ size_t piece_src[3][3]; // part (gate, up, down) x (trellis, suh, svh): offset in the region+ size_t piece_bytes[3][3];+ size_t dense_bytes = 0;+ // a GGUF's MXFP4 experts (lay.fmt == gg::kFmtMXFP4): matrix r (gate, up, down) of expert e lies at mx_off[r] ++ // e mx_stride[r] in file mx_file[r] - loaded from there, and read in place (the file's mapping) by the CPU lane+ int mx_file[3] = {-1, -1, -1};+ uint64_t mx_off[3] = {0, 0, 0}, mx_stride[3] = {0, 0, 0};+};++struct Src {+ int layer = 0, ratio = 1;+ int64_t cap = 0;+ // per device (nullptr where no layer reads it): rows of kKvFp4 (the compressed KV), kKvFp4p (the index keys)+ std::vector<uint8_t*> ckv, ik;+ uint8_t *h_ckv = nullptr, *h_ik = nullptr; // pinned: a window's new rows, on their way to the other devices+ cudaEvent_t ev_ckv = nullptr, ev_ik = nullptr;+ // threads: whether devices outside the source's group read it, and the host-mapped hand-off of a window's new rows+ // (ckv rows, then ik rows at kWin * hd) and its flag+ bool remote = false;+ uint8_t* x_stage = nullptr;+ unsigned* x_flag = nullptr;+ // pipelined chunks: a second staging slot (slot 1; slot 0 is x_stage), its flag, and the readers' acknowledgements+ // [slot][device] (32 apart); readers: the devices outside the source's group that read it+ uint8_t* x_stage1 = nullptr;+ unsigned* x_flag1 = nullptr;+ unsigned* x_ack = nullptr;+ std::vector<int> readers;+ uint8_t* stage(int slot) const { return slot ? x_stage1 : x_stage; }+ unsigned* flag(int slot) const { return slot ? x_flag1 : x_flag; }+};++// a host thread that enqueues one device's work (threads mode)+struct Worker {+ std::thread th;+ std::mutex mu;+ std::condition_variable cv;+ std::function<void()> job;+ bool busy = false, quit = false;+ void start() {+ th = std::thread([this] {+ for (;;) {+ std::function<void()> j;+ {+ std::unique_lock<std::mutex> lk(mu);+ cv.wait(lk, [&] { return quit || job != nullptr; });+ if (quit) return;+ j = std::move(job);+ job = nullptr;+ }+ j();+ {+ std::lock_guard<std::mutex> lk(mu);+ busy = false;+ }+ cv.notify_all();+ }+ });+ }+ void submit(std::function<void()> f) {+ {+ std::lock_guard<std::mutex> lk(mu);+ job = std::move(f);+ busy = true;+ }+ cv.notify_all();+ }+ void wait() {+ std::unique_lock<std::mutex> lk(mu);+ cv.wait(lk, [&] { return !busy; });+ }+ bool idle() {+ std::lock_guard<std::mutex> lk(mu);+ return !busy;+ }+ void stop() {+ {+ std::lock_guard<std::mutex> lk(mu);+ quit = true;+ }+ cv.notify_all();+ if (th.joinable()) th.join();+ }+};++struct Group {+ std::vector<int> mem; // device indices, member order (member 0 the lead)+ ArGroup ar, arb; // a window's all-reduces, a chunk's+ // the head group: the members' logits shares on their way to the lead, two slots (by seq), their flags and the+ // lead's acknowledgements (host-mapped)+ float* lg_stage = nullptr;+ unsigned *lg_flag = nullptr, *lg_ack = nullptr;+};++constexpr int kMaxL = 64; // layers (the trunk's and the drafter's) a window plan covers++// what a window's layers need from the host, decided before they run (threads enqueue the layers concurrently): the+// mailbox seq of each layer (async: the device publishes its routing), the indexer's selection sizes and the candidate+// blocks as each layer sees them, the window's hand-off seq+struct WinPlan {+ bool par = false, async = false;+ unsigned win = 0;+ int k = 0; // (a run of chunks) the chunk's index: its hand-off slot k % 2+ int snap_t = -1; // (a run of chunks) the chunk's row whose state before it the layers keep (-1: none)+ int eg_slot = -1; // (a run of chunks, layer-major) the Engram rows' slot (-1: k % 2)+ unsigned vl = 0; // (a window) bit t: token t is inside a picture's span (routed with the image bias)+ unsigned seq[kMaxL] = {};+ int topk[kMaxL][kWin] = {}, cand[kMaxL][kWin] = {};+};++} // namespace++// ------------------------------------------------------------------ the model+struct Model::Impl {+ Options o;+ Pack pk;+ Geo g;+ std::vector<Dev> devs;+ std::vector<Layer> L;+ std::vector<Src> srcs;+ std::vector<Group> groups;+ std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per device+ bool par = false; // threads mode (the windows: decode)+ mutable int all_gfx12 = -1; // rdna4(): -1 not asked yet+ // Every GPU the engine uses is a gfx1200 / gfx1201 (RDNA4: R9700, RX 9000) in a HIP build. The DeepSeek engine's share-based defaults (the small+ // chunk for multi-card tp runs, the whole-chunk slab, the small staging ring, the pinned copy at share >= 0.5) and the drafter's cumulative cut are+ // the 4 x R9700 measurements, so only RDNA4 takes them until another AMD part is tested (Strix Halo next).+ bool rdna4() const {+ if (all_gfx12 < 0) {+ bool yes = false;+#if defined(STRATA_USE_HIP)+ yes = !devs.empty();+ for (const auto& d : devs) {+ cudaDeviceProp prop{};+ yes = yes && cudaGetDeviceProperties(&prop, d.id) == cudaSuccess &&+ (std::strncmp(prop.gcnArchName, "gfx1200", 7) == 0 || std::strncmp(prop.gcnArchName, "gfx1201", 7) == 0);+ }+#endif+ all_gfx12 = yes ? 1 : 0;+ }+ return all_gfx12 == 1;+ }+ bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads asked+ unsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)+ // the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows+ int chunk = 0, chunk_min = 512;+ static constexpr bool kChunkedPromptDefault = true; // the layer-major prompt is the default (gate: KL 0.1685 vs 0.1686, 8K prompt 6x, decode within 2% - ds4 notes-dsv41-r97)+ static constexpr int kFlowChunkMin = 2; // chunk_min with every device's layers on the flow (see load)+ // --prefill-chunk at most (route() runs in kVlRows-row slabs). The default stays kVlRows: a 16384-token chunk+ // reads a 16K prompt's experts once instead of twice (V100: 461.8 -> 321.3 s, 736 -> 506 GB read) but its buffers+ // take ~1.9 GB of VRAM experts (745 -> 644 on a 32 GB V100) from every decoded token+ static constexpr int kMaxChunk = 4 * kVlRows;+ std::vector<std::unique_ptr<Chunk>> chunks; // per device+ size_t eg_cap = kWin; // tokens h_eg holds+ int tap_cap = kWin; // rows h_taps holds (a chunk seeds the drafter with its last 128)+ static constexpr int kYocoMin = 1024; // a long prompt's skip (forward_chunks): the fewest rows worth skipping+ bool logits_all = false; // the last feed was a chunk with logits for every row (chunk_logits)+ std::vector<float> chunk_logits;+ std::function<void(int64_t)> progress; // a run of chunks: the tokens done so far (the main thread, while it waits)+ // a Stop while a run of chunks goes through the layers (Model::set_cancel, asked while forward_chunks waits): one+ // GPU stops after the layer (or the layer's chunk) it is on (halt); several stop at a chunk every one of them+ // reaches - none starts chunk halt_k or later, halt_k one past the latest any started - so no device waits on a+ // hand-off that never comes (Project Maya v1.1.20)+ std::function<bool()> cancel;+ std::atomic<bool> halt{false};+ std::mutex halt_mu;+ int halt_k = INT_MAX, halt_seen = -1;+ bool halted = false; // the last run ended early on a Stop+ static constexpr const char* kHalted = "stopped";+ bool chunk_go(int k) { // chunk k may start on this device (several GPUs)+ std::lock_guard<std::mutex> lk(halt_mu);+ if (k >= halt_k) return false;+ halt_seen = std::max(halt_seen, k);+ return true;+ }+ bool halt_now() const { return devs.size() == 1 && halt.load(std::memory_order_acquire); }+ // a run's early end on device di: the expert reads still in flight land first (they count down items the next run+ // clears), then its streams are idle+ bool halt_end(int di, std::string& err) {+ Chunk& c = *chunks[(size_t) di];+ cudaSetDevice(devs[(size_t) di].id);+ if (c.flow)+ for (size_t i = c.base; i < c.n_read; ++i)+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);+ if (c.cs != nullptr) cudaStreamSynchronize(c.cs);+ cudaStreamSynchronize(devs[(size_t) di].st);+ err = kHalted;+ return false;+ }+ std::vector<cudaEvent_t> chunk_done; // on the last layer's device: chunk k through every layer+ std::unique_ptr<std::atomic<int>[]> chunk_rec; // chunk k's event recorded+ // a prompt the flow could take: the flow or windows of 16, whichever this PC runs it faster (pick_path) - this+ // PC's speeds measured at load (calibrate_paths), each path's predictions scaled by how long it really took+ struct PathModel {+ bool on = false;+ double dma = 0, copy = 0, disk = 0, cpu = 0, cpu_cold = 0; // bytes/s (disk, cpu_cold 0: not measured)+ // the flow's taken / predicted so far [0] by the prompt's size class (2-31, 32-127, 128-511, 512 tokens and+ // more: a short one's fixed costs weigh more), and how many prompts set it ([1] unused)+ double k[2][4] = {{1, 1, 1, 1}, {1, 1, 1, 1}};+ int64_t kn[2][4] = {};+ double win_s = 0; // windows: seconds a window took, so far+ int64_t win_n = 0;+ int last_cls = 0, last_nw = 0; // the last pick's size class, its windows+ double m = 1; // the picks' spread, against the profile's+ int64_t n_m = 0;+ int last = -1; // the last pick: 1 the flow, 0 windows+ double last_pred = 0; // its prediction (s, before its k)+ } pm;+ static constexpr int kPickMax = 2048; // (from here on the flow: windows' work grows with every 16 tokens)+ static constexpr double kPickRes = 0.9; // (this share of the CPU-lane experts in RAM: windows if predicted faster)+ std::atomic<int> lm_layers{0}; // (layer-major) the layers every chunk went through+ WinPlan wplan;+ int head_dev = 0;+ x3::Mat head;+ float* norm_w = nullptr; // on the head device+ // the drafter: main_proj / main_norm (on its first block's device), its final norm, the markov head and the+ // confidence head (on the head device: the drafts' logits come from the trunk's head)+ x3::Mat d_main_proj;+ float *d_main_norm = nullptr, *d_norm = nullptr, *d_conf = nullptr;+ uint16_t *d_mk_embed = nullptr, *d_mk_head = nullptr; // BF16 [vocab][rank], F16 [vocab][rank]+ uint8_t* d_mk_head_q8 = nullptr; // the markov head as Q8_0 blocks (the same bias, half the bytes a step)+ // a head group of two or more: the markov steps over its members (each its rows of the bias and an argmax of its+ // share, the global one through host-mapped slots [step][member] {value, index, seq}) in place of the lead's whole+ // rows - E host-mapped (each member reads the picked token's row), a draft's seq in mk_seq+ bool mk_split = false;+ uint16_t* h_mk_embed = nullptr;+ unsigned* mk_slots = nullptr;+ unsigned mk_seq = 0;+ int d_dev = 0; // the drafter's first block's device+ bool spec_win = false; // the window being run is a draft's verification (layer() keeps backups)+ float* h_taps = nullptr; // pinned [kWin][targets][dim]: the last window's taps+ cudaEvent_t ev_seed = nullptr; // the drafter's seeding read h_taps: the next window's taps wait for it+ float* d_confv = nullptr; // on the head device+ int* d_ints = nullptr; // on the head device: drafts in, picks out+ float conf_threshold = std::getenv("DS4_DSPARK_CONF") ? (float) std::atof(std::getenv("DS4_DSPARK_CONF")) : 0.5f;+ // a sampled round's drafts: the distribution q each one was drawn from (its kept list, on the head device) for+ // verify()'s rule - a point at the draft for the drafter's argmax; and the chain they were drawn with (verify()+ // checks it is its own)+ strata::dsv41k::SpecList* d_specq = nullptr;+ std::vector<strata::dsv41k::SpecList> h_specq = std::vector<strata::dsv41k::SpecList>(kWin);+ bool drafts_q = false;+ float dq_temp = 0.0f, dq_top_p = 1.0f, dq_min_p = 0.0f;+ int dq_top_k = 0;+ uint64_t dq_seed = 0, dq_counter = 0;+ // (tests) DS4_DSPARK_SAMPLE=1: a sampled request's drafts drawn from the drafter's distribution through the+ // request's chain, as DeepSeek's reference does (else its argmax: the same acceptance on Uranus, 31-32% of 5+ // drafts a round either way); DS4_DSPARK_QTEMP=f: those draws at f x the request's temperature (any q keeps the+ // rule exact); DS4_DSPARK_MAX=n: at most n drafts a round; DS4_DSPARK_DUMP=file: the drafter's inputs and outputs+ // (tools/ds4/dspark_ref.py checks them against DeepSeek's checkpoint)+ bool sample_drafts = std::getenv("DS4_DSPARK_SAMPLE") != nullptr && std::atoi(std::getenv("DS4_DSPARK_SAMPLE")) != 0;+ float q_temp_scale = std::getenv("DS4_DSPARK_QTEMP") ? (float) std::atof(std::getenv("DS4_DSPARK_QTEMP")) : 1.0f;+ int max_drafts = std::getenv("DS4_DSPARK_MAX") ? std::atoi(std::getenv("DS4_DSPARK_MAX")) : 0;+ std::FILE* dump_d = std::getenv("DS4_DSPARK_DUMP") ? std::fopen(std::getenv("DS4_DSPARK_DUMP"), "wb") : nullptr;+ float* h_dump = nullptr; // (the dump's pinned copies: each block's stream, each block's ring)+ uint16_t* h_dump_ring = nullptr;++ uint16_t* embed_pinned = nullptr;+ std::vector<float> freqs_win, freqs_comp;+ std::vector<std::unique_ptr<Pool>> pools; // one per NUMA node, its workers on that node's cores+ std::vector<std::vector<int>> cores; // each node's physical cores+ int NH = 1; // the CPU lane's parts of an expert (the NUMA nodes)+ // the CPU lane's experts read in place from the model's files (one part, the files' layout; the page cache keeps+ // what fits) instead of copied into RAM, and how much of them the load reads ahead (the most-routed first)+ bool file_experts = false;+ uint64_t warm_bytes = 0;+ static constexpr uint64_t kRamMargin = 8ull << 30; // RAM left to the rest (the system, the page cache's churn)+ uint8_t* file_part(const Layer& l, int e) const {+ return pk.maps[(size_t) l.region_file[(size_t) e]].first + l.region_off[(size_t) e];+ }+ // host side of the CPU lane, a window's rows+ float *h_x = nullptr, *h_w = nullptr, *h_part = nullptr;+ int* h_idx = nullptr;++ int* h_idx_a = nullptr; // (Metal) the lookahead's guessed picks, 2 x kPicks+ uint8_t* h_eg = nullptr; // the window's Engram rows: [engram layer][token] of [cols][hd] then the scales+ // the Engram rows' reader (its own threads and file handles), the rows already fetched, the window's batches (one+ // an Engram layer: asked for when the window's tokens are known, collected right before that layer)+ EgReader eg_io;+ EgRowCache eg_cache;+ std::mutex eg_cache_mu;+ std::vector<std::unique_ptr<EgBatch>> eg_win;+ // Engram layer li's embed rows, as the pack stores them: 0 MXFP8 ([scale, 32 codes] blocks, 33 bytes a 32), 1+ // ggml Q6_K (210 bytes a 256), 2 ggml IQ4_NL (18 bytes a 32: an fp16 d and 16 nibble bytes), 3 an old pack's FP8+ // codes with their E8M0 scales in a tensor of their own (see eg_issue)+ std::vector<uint8_t> eg_fmt;+ std::atomic<int64_t> eg_rows_n{0}, eg_hits_n{0}, eg_reads_n{0}, eg_wait_ns{0}, eg_io_ns{0};+ uint16_t* h_emb = nullptr; // the window's embedding rows+ std::vector<float> c_xh, c_xsb, c_acc, c_xhd, c_xsd, c_accd, c_yd;+ // the state+ int64_t pos = 0;+ std::vector<int32_t> toks, ctoks;+ struct Check { // the state at a position: every window ring, the compressors' partial groups+ int64_t pos = -1;+ std::vector<std::vector<uint8_t>> win;+ std::vector<std::vector<float>> comp;+ };+ static constexpr int kChecks = 4;+ Check ck[kChecks]; // the end of the last prompt, the position before it, the anchor (the position+ // before the end of the first prompt of the run of prompts that each continued+ // the last in place: a chat template that drops earlier turns' reasoning at a new+ // user message re-renders the history from about there) and the prefix (where the+ // server said the prompt's shared start - system prompt and tools - ends: another+ // agent behind the same start goes back to it)+ bool slot_held = false; // slot_save took this state and nothing has changed it since+ std::vector<uint32_t> prof; // [nl][n_exp] routed-expert hits+ Stats st;+ int cur_topk_n[kWin] = {}, cur_cand_n[kWin] = {}; // what the latest indexer / candidate source left, per query+ // (plan_window's: the windows' plans are made in order)+ int n_logits = 0; // the last forward's logit rows+ // the decode path (windows of one): the device publishes each layer's routing into reqs, the service thread+ // answers in resps with the CPU lane's part, and the device waits for the answer only before the expert sum+ Req* reqs = nullptr;+ Resp* resps = nullptr;+ unsigned seq_issued = 0;+ std::atomic<int> layer_of[kRing];+ std::thread service;+ std::atomic<bool> quit{false};+ std::thread store_bg; // the RAM store's fill beyond the first 48 GB, read after the model is ready (preload_store)++#if defined(STRATA_METAL)++ // (the async decode has the GPU wait on flags the CPU lane sets while it runs: the Metal queue cannot be waited on in the++ // middle of a command buffer, so a decode step takes the host-routed path, as the prompt windows do)++ bool sync_decode = std::getenv("DS4_ASYNC") == nullptr;++#else+ bool sync_decode = std::getenv("DS4_SYNC") != nullptr;++#endif+ // a window of one through moe_gate_up / moe_down (DS4_MOE_TOKEN=0: win_experts, whose down rows run+ // before the wait for the CPU lane - the same bits, no faster)+ bool moe_token_path = std::getenv("DS4_MOE_TOKEN") == nullptr || std::atoi(std::getenv("DS4_MOE_TOKEN")) != 0;+ int async_max_t = std::getenv("DS4_ASYNC_T") ? std::atoi(std::getenv("DS4_ASYNC_T")) : 8; // windows up to this+ // lookahead: while layer L runs, the device guesses the next layer's picks (its gate on L's FFN input, the same+ // choice) and publishes them with L's routing; the CPU lane asks for the guessed experts it will read from the files+ // at once (a thread of its own, os::prefetch), so the SSD reads them while L computes instead of on the next+ // layer's critical path. Only the page cache sees it - the outputs are the same. DS4_AHEAD=0: off, 2: also the+ // layer after (from L's input too); DS4_AHEAD_EXTRA: guesses past the next layer's top k (< 0: fewer - the guess's+ // picks by rank on Sun: 97 91 81 70 57 41 %, past the top 6 29 20 %: reads wasted on an SSD-bound PC).+ int ahead_mode = std::getenv("DS4_AHEAD") ? std::atoi(std::getenv("DS4_AHEAD")) : 1;+ int ahead_extra = std::getenv("DS4_AHEAD_EXTRA") ? std::atoi(std::getenv("DS4_AHEAD_EXTRA")) : 0;+ // (DS4_AHEAD_ORDER=0: the CPU lane computes a layer's experts in expert order, not the ones asked for early first)+ bool ahead_order = std::getenv("DS4_AHEAD_ORDER") == nullptr || std::atoi(std::getenv("DS4_AHEAD_ORDER")) != 0;+ std::vector<uint8_t> ahead_fl; // [layer][1024]: 1 the top-k guess, 2 any guess, a layer before; 4 two layers before;+ // 8 asked for (prefetch)+ std::vector<uint8_t> ahead_rank; // [layer][1024]: 1 + the expert's rank in the guess a layer before (0: not guessed)+ std::vector<unsigned> ahead_for; // [layer]: the request (seq) its flags are for+ std::thread pf_th;+ std::mutex pf_mu;+ std::condition_variable pf_cv;+ std::vector<std::pair<const void*, size_t>> pf_q;+ // the expert cache: swaps prepared on threads of their own (DS4_CACHE_THREADS, 1 by default - 4 measured no faster+ // on one V100 - more for a PC to try: each its staging buffers and+ // copy streams), switched between windows+ struct Swap {+ int layer = 0, in = -1, out = -1;+ int ram = -1; // (copy_mode, `in` held in RAM) the spare slot `out` is copied into from VRAM+ };+ static constexpr int kMaxSwaps = 8;+ // the swap rule: a CPU-lane expert in when its heat passes the coldest VRAM one's x margin + bonus; heat x decay+ // every 256 tokens; at most `inflight` swaps under way (DS4_CACHE_MARGIN / _BONUS / _DECAY / _INFLIGHT)+ static float env_f(const char* k, float d) { return std::getenv(k) ? (float) std::atof(std::getenv(k)) : d; }+ const float c_margin = env_f("DS4_CACHE_MARGIN", 1.25f), c_bonus = env_f("DS4_CACHE_BONUS", 1.0f),+ c_decay = env_f("DS4_CACHE_DECAY", 0.9f);+ const int c_inflight = std::max(1, (int) env_f("DS4_CACHE_INFLIGHT", (float) kMaxSwaps));+ // the RAM tier (a GGUF's in-place experts, as GLM's: the hottest in VRAM, the next hottest here, the coldest in the+ // files): RAM slots of one expert blob each, filled at load with the hottest experts VRAM does not hold, so the+ // CPU lane reads them from RAM; between windows (cache_tick) a hotter expert from the files takes the place of the+ // layer's coldest held one - read into a free slot by ram_th, switched while nothing reads them. Opt-in+ // (STRATA_DS4_RAM_TIER=1): on a V100 with 64 GB of RAM it was slower than the page cache alone (1.4 against 2.2-2.4+ // tokens/s sampled) - the cache follows a conversation's experts at once, the tier's moves lag behind.+ struct RamSwap {+ int layer = 0, in = -1, out = -1, slot = -1;+ };+ std::vector<uint8_t*> ram_slots, ram_chunks;+ std::vector<size_t> ram_chunk_bytes;+ bool ram_pinned = false; // the slots are pinned memory (the prompt path DMAs them from where they lie)+ size_t flow_part[3] = {0, 0, 0}; // the flow's staging room for each part: its largest over the trunk's layers+ std::vector<int> ram_free;+ std::deque<RamSwap> ram_queue;+ std::vector<RamSwap> ram_ready;+ std::condition_variable ram_cv;+ std::thread ram_th;+ int ram_inflight = 0, ram_held = 0;+ bool ram_tier = std::getenv("STRATA_DS4_RAM_TIER") != nullptr && std::atoi(std::getenv("STRATA_DS4_RAM_TIER")) != 0;+ // STRATA_DS4_DISK_SKIP_W (experimental opt-in, CHANGES THE ANSWERS; 0 = off): on the small windows (decode, a+ // draft's check - never the prompt path) the CPU lane leaves out a routed expert whose share of its token's route+ // weights is below this and whose pages are not in RAM now (the page cache or the RAM tier; os::in_ram - Linux every+ // page, Windows sampled pages of the working set), so a token waits on the SSD only for the experts that weigh; the+ // weight it carried is lost (no renormalisation). On layers from STRATA_DS4_DISK_SKIP_FROM (0) on. An expert left+ // out still counts as used (the expert cache, the profile).+ const float disk_skip_w = [] {+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_W");+ return v != nullptr ? std::max(0.0f, (float) std::atof(v)) : 0.0f;+ }();+ const int disk_skip_from = [] {+ const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");+ return v != nullptr ? std::max(0, std::atoi(v)) : 0;+ }();+ // the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)+ std::unique_ptr<ExpertStore> store;+ uint8_t* zc_stage_buf = nullptr; // device: k staging slots for the store's experts of a decode token+ size_t zc_stride = 0;+ bool zc_copy = true; // DS4_STORE_STAGE=0: the expert kernels read the arena over PCIe themselves+ bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM now+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;+ if (l.lay.fmt == gg::kFmtMXFP4) {+ for (int r = 0; r < 3; ++r)+ if (!os::in_ram(mx_mat(l, e, r), mx_part_bytes(l, r))) return false;+ return true;+ }+ return !file_experts || os::in_ram(file_part(l, e), l.blob);+ }+ // a GGUF's CPU-lane experts copied into RAM (copy_experts, after VRAM took its share: --mmap-experts off - auto reads+ // them in place; DS4_COPY_GB=<n>: the hottest n GB, the rest in place - static, no moves between+ // them): RAM slots (l.ram, as the tier's) of NH parts, part X on NUMA node X with rows [X n / NH, (X + 1) n / NH) of+ // each matrix at cp_lay[layer][r] - whole rows, so the split never cuts a GGUF block (MXFP4's 32, the IQ / K types'+ // 256 values: blocks run along a row); one spare slot a layer for the cache's swaps (the expert VRAM gives back takes+ // the RAM place of the one it takes in)+ // Windows' commit on an integrated GPU: its allocations are charged to the commit (RAM + page file) like any private+ // memory - measured on a Radeon 8065S with 160 GiB of its own: the system's free commit fell by each one, whatever+ // its size (one above ~3 GiB also counts as the process's private bytes, a smaller one does not, the commit the+ // same), so a page file at least the GPU's share is what lets it hold experts at all. The experts it holds+ // (place_experts) and the copies stay within what Windows can still commit, less max(8 GB, 10% of the RAM) for the+ // other programs - commit_kept: the experts that kept off the GPU, pagefile_hint: the page file (GB) that would hold+ // them (the log's WARNING, a NOTE and INFO for the server)+ double commit_kept = 0, pagefile_hint = 0;+ double commit_margin() const {+ const os::Commit c = os::commit();+ return std::max(8.0 * (double) (1ull << 30), c.ram == ~uint64_t{0} ? 0.0 : 0.1 * (double) c.ram);+ }+ // what may still be committed beside the margin (~inf: nothing limits it; DS4_COMMIT_CAP_GB: at most that, tests)+ double commit_room() const {+ const os::Commit c = os::commit();+ double avail = c.avail == ~uint64_t{0} ? 1e300 : (double) c.avail;+ if (const char* v = std::getenv("DS4_COMMIT_CAP_GB")) avail = std::min(avail, std::atof(v) * 1e9);+ return avail >= 1e300 ? 1e300 : std::max(0.0, avail - commit_margin());+ }+ // a device whose allocations Windows charges to the commit: an integrated GPU (DS4_COMMIT_CAP_GB: any, tests)+ bool commit_charged(const Dev& d) const {+ if (std::getenv("DS4_COMMIT_CAP_GB") != nullptr) return true;+#if defined(_WIN32)+ int v = 0;+ if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) != cudaSuccess) {+ (void) cudaGetLastError();+ return false;+ }+ return v != 0;+#else+ (void) d;+ return false;+#endif+ }+ bool copy_mode = false;+ std::vector<std::array<uint8_t*, 2>> ram_part; // slot -> its parts+ std::vector<std::array<size_t, 4>> cp_lay; // layer -> part offsets of gate, up, down rows; a part's bytes+ std::vector<int> copy_spare; // layer -> its spare slot (-1: none)+ std::mutex sw_mu;+ std::condition_variable sw_cv;+ std::deque<Swap> sw_queue;+ std::vector<Swap> sw_ready;+ struct SwapBufs {+ uint8_t *a = nullptr, *b = nullptr, *h = nullptr;+ std::vector<cudaStream_t> st; // a copy stream a device+ };+ std::vector<SwapBufs> sw_bufs;+ std::vector<std::thread> cache_ths;+ int sw_inflight = 0;+ int64_t decay_tokens = 0;+ bool check_swaps = std::getenv("DS4_CHECK_SWAPS") != nullptr;+ std::vector<float> host_logits;+ std::vector<void*> host_allocs;+ std::FILE* dump_h = std::getenv("DS4_DUMP_H") ? std::fopen(std::getenv("DS4_DUMP_H"), "wb") : nullptr;+ // DS4_DUMP_L: (threads) every layer's residual streams as records [layer, p0, T, T x hc x dim floats], window or+ // chunk path (tests: where the two part)+ std::FILE* dump_l = std::getenv("DS4_DUMP_L") ? std::fopen(std::getenv("DS4_DUMP_L"), "wb") : nullptr;++ // DS4_DUMP_S: the sub-steps inside the layers named by DS4_DUMP_S_LAYERS (default 0,1,2,3), records++ // [char name[16], int64 il, p0, T, nbytes][nbytes raw bytes] (window path, the same on CUDA and Metal)++ std::FILE* dump_s = std::getenv("DS4_DUMP_S") ? std::fopen(std::getenv("DS4_DUMP_S"), "wb") : nullptr;++ std::vector<int> dump_s_layers = [] {++ std::vector<int> v;++ const char* e = std::getenv("DS4_DUMP_S_LAYERS");++ std::string str = e ? e : "0,1,2,3";++ size_t a = 0;++ while (a < str.size()) {++ size_t b = str.find(',', a);++ if (b == std::string::npos) b = str.size();++ if (b > a) v.push_back(std::atoi(str.substr(a, b - a).c_str()));++ a = b + 1;++ }++ return v;++ }();+ // DS4_DUMP_R: the chunk path's routing, a record a layer [layer, p0, T, k, n_exp, n_vs, n_slots, kPicks] (int64),+ // then the picks' experts [T kPicks] (the first k of each token's), each expert's virtual slot [n_exp], the+ // slots' row bounds [n_vs + 1], the sorted rows' tokens [T k] (int32), the gate's logits [T n_exp] and its bias+ // [n_exp] (F32)+ std::FILE* dump_r = std::getenv("DS4_DUMP_R") ? std::fopen(std::getenv("DS4_DUMP_R"), "wb") : nullptr;+ // DS4_DUMP_Y=<layer>:<file> (layer -1: every layer): its expert sets on the chunk path, a record a piece [layer,+ // row0, rows, n_exp, wbase kind (0 VRAM, 1 ring), max_rows] (int64), its bounds [n_exp + 1], its h rows [rows ff]+ // (F16) and y rows [rows dim] (F32)+ int dump_y_layer = std::getenv("DS4_DUMP_Y") ? std::atoi(std::getenv("DS4_DUMP_Y")) : -1;+ std::FILE* dump_y = [] {+ const char* v = std::getenv("DS4_DUMP_Y");+ const char* f = v != nullptr ? std::strchr(v, ':') : nullptr;+ return f != nullptr ? std::fopen(f + 1, "wb") : nullptr;+ }();+ std::mutex dump_mu;+ // DS4_PROF: the GPU's time a decoded token by stage - events in stream order at a layer's start, after its+ // attention, after the routing and the shared expert, after the experts' gate / up, after the wait for the CPU+ // lane, after the down pass and the post - read after the token's sync, each the time since the mark before it+ bool prof_on = std::getenv("DS4_PROF") != nullptr;+ // DS4_PRED_DUMP=file (predictor data, the chunked prompt path): per layer and chunk u8 layer, u16 T, T x 384 fp16 router+ // logits, T x 256 fp16 projection of the router input. DS4_XIN_DUMP=file (expert similarity inputs): u8 layer, u16 T,+ // T x D fp32 router inputs, the first DS4_XIN_N (1024) tokens of each layer.+ std::FILE* pred_f = std::getenv("DS4_PRED_DUMP") != nullptr ? std::fopen(std::getenv("DS4_PRED_DUMP"), "wb") : nullptr;+ std::FILE* xin_f = std::getenv("DS4_XIN_DUMP") != nullptr ? std::fopen(std::getenv("DS4_XIN_DUMP"), "wb") : nullptr;+ int xin_n = std::getenv("DS4_XIN_N") ? std::atoi(std::getenv("DS4_XIN_N")) : 1024;+ std::array<int, 64> xin_got{};+ std::FILE* trace_f = std::getenv("DS4_TRACE") != nullptr ? std::fopen(std::getenv("DS4_TRACE"), "wb") : nullptr;+ struct ProfMark {+ const Dev* d;+ cudaEvent_t ev;+ int stage;+ };+ std::mutex prof_mu;+ std::vector<ProfMark> prof_marks;+ std::map<int, std::vector<cudaEvent_t>> prof_pool; // by device: an event belongs to the device it was made on+ bool prof_window = false; // (decode windows only: T == 1)+ // ... and a run of chunks' (prof_chunk): a layer's start, after its attention, after the routing and the shared+ // expert, after its experts (the streamed ones' waits for their reads in it); printed after the run+ bool prof_chunk = false;+ // DS4_PROF stages of the chunked prompt (finer ones than the decode's 0..5): 0 between layers, 1 attention rest (wq_b, wo,+ // hc_post), 2 router-side tail (shared expert), 3 experts (+ waits), 4 Engram, 5 attn mHC (mix, split, norm), 6 attention+ // projections (wq_a, wkv, compressor), 7 compressor + indexer, 8 q_b + rope, 9 sparse attention, 10 ffn mHC + router + sort+ double cprof_ms[16] = {};+ // DS4_PROF: the experts' kernels alone, by type (events around the gate/up and down calls of run_set)+ struct EkMark {+ cudaEvent_t e0, e1, e2;+ int tg, td, nrows, nexp, maxrows, dev, ff;+ };+ std::vector<EkMark> ek_marks;+ std::map<int, std::vector<cudaEvent_t>> ek_pool; // by device+ cudaEvent_t ek_ev() {+ int cur = 0;+ cudaGetDevice(&cur);+ std::vector<cudaEvent_t>& pl = ek_pool[cur];+ if (!pl.empty()) {+ cudaEvent_t e = pl.back();+ pl.pop_back();+ return e;+ }+ cudaEvent_t e = nullptr;+ cudaEventCreate(&e);+ return e;+ }+ void prof_mark(const Dev& d, int stage) {++ if (std::getenv("DS4_TLOG")) { // the host's own phase times (what the GPU meters do not see)++ static double t_last = 0, acc[5] = {0, 0, 0, 0, 0}; static long n = 0; static bool reg = false;++ if (!reg) {++ reg = true;++ std::atexit([] {++ std::fprintf(stderr,++ "tlog: %ld layers | host ms: attn %.0f moe_pre %.0f gate_up %.0f cpu_wait %.0f down+post %.0f\n",++ n, acc[0], acc[1], acc[2], acc[3], acc[4]);++ });++ }++ const double t = now_ms();++ if (t_last > 0) {++ const int b = stage == 0 ? 4 : stage - 1;++ acc[b >= 0 && b < 5 ? b : 4] += t - t_last;++ }++ t_last = t;++ if (stage == 3) ++n;++ }+ if (!prof_window && !prof_chunk) return;+ std::lock_guard<std::mutex> lk(prof_mu);+ int cur = 0;+ cudaGetDevice(&cur);+ cudaSetDevice(d.id); // (events of another device are an "invalid resource handle")+ cudaEvent_t e = nullptr;+ std::vector<cudaEvent_t>& pool = prof_pool[d.id];+ if (pool.empty()) cudaEventCreate(&e);+ else {+ e = pool.back();+ pool.pop_back();+ }+ cudaEventRecord(e, d.st);+ cudaSetDevice(cur);+ prof_marks.push_back({&d, e, stage});+ }+ void prof_collect() {+ if (!prof_window && !prof_chunk) return;+ std::lock_guard<std::mutex> lk(prof_mu);+ std::vector<std::pair<const Dev*, cudaEvent_t>> last;+ for (const ProfMark& m : prof_marks) {+ auto it = std::find_if(last.begin(), last.end(), [&](const auto& x) { return x.first == m.d; });+ if (it != last.end()) {+ float ms = 0.0f;+ cudaSetDevice(m.d->id);+ if (cudaEventElapsedTime(&ms, it->second, m.ev) == cudaSuccess)+ (prof_chunk ? cprof_ms : st.prof_ms)[m.stage] += ms;+ it->second = m.ev;+ } else {+ last.push_back({m.d, m.ev});+ }+ }+ for (const ProfMark& m : prof_marks) prof_pool[m.d->id].push_back(m.ev);+ prof_marks.clear();+ if (!prof_chunk) ++st.prof_tokens;+ }+ void dump_layer(Dev& d, const float* h, int il, int64_t p0, int T) {+ std::vector<float> v((size_t) T * g.hc * g.dim);+ cudaSetDevice(d.id);++#if defined(STRATA_METAL)++ ds4_metal_flush(); // (the residual stream is read back here: commit the batched dispatches first)++ cudaDeviceSynchronize();++#endif+ cudaMemcpyAsync(v.data(), h, v.size() * 4, cudaMemcpyDeviceToHost, d.st);+ cudaStreamSynchronize(d.st);+ std::lock_guard<std::mutex> lk(dump_mu);+ const int64_t hdr[3] = {il, p0, T};+ std::fwrite(hdr, 8, 3, dump_l);+ std::fwrite(v.data(), 4, v.size(), dump_l);+ std::fflush(dump_l);+ }+++ bool dump_s_on(int il) const {++ if (dump_s == nullptr) return false;++ for (int x : dump_s_layers)++ if (x == il) return true;++ return false;++ }++ bool dump_s_name(const char* name) const { // DS4_DUMP_S_NAMES=a,b,c: only these points (default all)++ static const std::string names = [] { const char* e = std::getenv("DS4_DUMP_S_NAMES"); return std::string(e ? "," : "") + (e ? e : "") + (e ? "," : ""); }();++ return names.empty() || names.find(std::string(",") + name + ",") != std::string::npos;++ }++ void dump_sub(Dev& d, const char* name, int il, int64_t p0, int T, const void* dev, size_t nbytes) {++ if (!dump_s_on(il) || !dump_s_name(name)) return;++ std::vector<uint8_t> v(nbytes);++ cudaSetDevice(d.id);++#if defined(STRATA_METAL)++ ds4_metal_flush();++ cudaDeviceSynchronize(); // (the batched dispatches have finished before the read)++#endif++ cudaMemcpyAsync(v.data(), dev, nbytes, cudaMemcpyDeviceToHost, d.st);++ cudaStreamSynchronize(d.st);++ dump_sub_host(name, il, p0, T, v.data(), nbytes);++ }++ void dump_sub_host(const char* name, int il, int64_t p0, int T, const void* host, size_t nbytes) {++ if (!dump_s_on(il) || !dump_s_name(name)) return;++ std::lock_guard<std::mutex> lk(dump_mu);++ char nm[16] = {};++ std::strncpy(nm, name, 15);++ const int64_t hdr[4] = {il, p0, T, (int64_t) nbytes};++ std::fwrite(nm, 1, 16, dump_s);++ std::fwrite(hdr, 8, 4, dump_s);++ std::fwrite(host, 1, nbytes, dump_s);++ std::fflush(dump_s);++ }+++ ~Impl() {+ quit.store(true);+ if (warm_bg.joinable()) warm_bg.join(); // (the warm-up uses the BLAS handles destroyed below)+ eg_io.stop(); // (its threads write into h_eg)+ for (auto& w : workers) w->stop();+ for (size_t i = 0; i < chunks.size(); ++i)+ if (chunks[i] && chunks[i]->blas) {+ cudaSetDevice(devs[i].id);+ cublasDestroy(chunks[i]->blas);+ }+ quit.store(true);+ if (service.joinable()) service.join();+ if (store_bg.joinable()) store_bg.join(); // (the store's background fill: it checks quit between its pieces)+ {+ std::lock_guard<std::mutex> lk(pf_mu);+ }+ pf_cv.notify_all();+ if (pf_th.joinable()) pf_th.join();+ {+ std::lock_guard<std::mutex> lk(sw_mu);+ }+ sw_cv.notify_all();+ ram_cv.notify_all();+ for (std::thread& t : cache_ths)+ if (t.joinable()) t.join();+ if (ram_th.joinable()) ram_th.join();+ for (size_t i = 0; i < ram_chunks.size(); ++i) {+ if (ram_pinned) (void) cudaHostUnregister(ram_chunks[i]);+ os::anon_free(ram_chunks[i], ram_chunk_bytes[i]);+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ for (void* p : d.allocs) cudaFree(p);+ if (d.st) cudaStreamDestroy(d.st);+ if (d.ev) cudaEventDestroy(d.ev);+ }+ for (void* p : host_allocs) cudaFreeHost(p);+ for (Layer& l : L)+ for (uint8_t* b : l.host)+ if (b != nullptr) {+ (void) b; // host blobs live in one mapping per layer, unmapped below+ }+ for (auto& m : host_maps) {+ if (chunk > 0) cudaHostUnregister(m.first);+ os::anon_free(m.first, m.second);+ }+ }+ std::vector<std::pair<void*, size_t>> host_maps;++ template <class T>+ bool dalloc(Dev& d, T*& p, size_t n, std::string& err, size_t* acct = nullptr) {+ void* v = nullptr;+ cudaSetDevice(d.id);+ const size_t b = align256(n * sizeof(T));+ if (cudaMalloc(&v, b) != cudaSuccess) {+ err = "cudaMalloc " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";+ return false;+ }+ // zeroed before anything lands in it: the loads copy in on the default stream, which the non-blocking d.st+ // does not order against - a zeroing still queued there could wipe the weights just copied+ if (cudaMemsetAsync(v, 0, b, d.st) != cudaSuccess || cudaStreamSynchronize(d.st) != cudaSuccess) {+ cudaFree(v);+ err = "zeroing " + std::to_string(b >> 20) + " MiB on CUDA" + std::to_string(d.id) + " failed";+ return false;+ }+ d.allocs.push_back(v);+ if (acct) *acct += b;+ else d.scratch += b;+ p = (T*) v;+ return true;+ }+ static Layer& member(Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }+ static const Layer& member(const Layer& l, int m) { return m == 0 ? l : l.mates[(size_t) m - 1]; }+ // bytes of a VRAM slot of l's experts (a member's part of each, in a group)+ static size_t unit(const Layer& l) { return l.nm > 1 ? l.hblob : l.blob; }+ // the members of a group of nm that take heads: the most the heads and the wo_a groups split among evenly (64+ // heads and 8 groups: 9 -> 8, 7 or 6 -> 4, 5 -> 4, 3 -> 2); the first members the rest, only their FFN share+ int head_members(int nm) const {+ int na = nm;+ while (na > 1 && (g.heads % na != 0 || g.o_groups % na != 0)) --na;+ return na;+ }+ // host-mapped memory (the same address on the host and every device), zeroed+ template <class T>+ bool hmap(T*& p, size_t n, std::string& err) {+ void* v = nullptr;+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {+ err = "host-mapped memory (" + std::to_string((n * sizeof(T)) >> 20) + " MiB)";+ return false;+ }+ std::memset(v, 0, n * sizeof(T) + 64);+ host_allocs.push_back(v);+ p = (T*) v;+ return true;+ }+ template <class T>+ bool halloc(T*& p, size_t n, std::string& err) {+ void* v = nullptr;+ if (cudaHostAlloc(&v, n * sizeof(T) + 64, cudaHostAllocPortable) != cudaSuccess) {+ err = "cudaHostAlloc " + std::to_string((n * sizeof(T)) >> 20) + " MiB failed";+ return false;+ }+ host_allocs.push_back(v);+ p = (T*) v;+ return true;+ }++ // ---- weights+ bool upload(Dev& d, const Tensor& t, void* dst, std::string& err) {+ std::vector<uint8_t> tmp(t.bytes);+ if (!pk.read(t, tmp.data())) {+ err = "read failed";+ return false;+ }+ cudaSetDevice(d.id);+ CK(cudaMemcpy(dst, tmp.data(), t.bytes, cudaMemcpyHostToDevice));+ return true;+ }+ // a dense matrix in a ggml block type (x3::Mat fmt kFmtGG, row = k / 256 blocks): whole, or part of its rows / inputs+ bool mat_gg(Dev& d, const std::string& pre, const Tensor& t, int ggt, x3::Mat& m, std::string& err, size_t* acct, int part,+ int np, bool cols) {+ if (t.shape.size() != 2 || t.shape[1] % 256 != 0) {+ err = pre + ": a " + t.dtype + " tensor that is not a matrix of whole blocks";+ return false;+ }+ int n = (int) t.shape[0], k = (int) t.shape[1];+ const size_t rb = gg::type_row_bytes(ggt, k), bb = rb / ((size_t) k / 256);+ std::vector<uint8_t> w((size_t) n * rb);+ if (t.bytes != w.size() || !pk.read(t, w.data())) {+ err = pre + ": read failed";+ return false;+ }+ if (np > 1) {+ const int whole = cols ? n : k, wd = whole / np;+ if (whole % np != 0 || wd % (cols ? 1 : 256) != 0) {+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " + std::to_string(np) + " parts";+ return false;+ }+ std::vector<uint8_t> p;+ if (cols) {+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));+ n = wd;+ } else {+ const size_t pb = (size_t) (wd / 256) * bb, off = (size_t) (part * wd / 256) * bb;+ p.resize((size_t) n * pb);+ for (size_t r = 0; r < (size_t) n; ++r) std::memcpy(p.data() + r * pb, w.data() + r * rb + off, pb);+ k = wd;+ }+ w.swap(p);+ }+ uint8_t* dp = nullptr;+ if (!dalloc(d, dp, w.size(), err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));+ m = x3::Mat{};+ m.fmt = gg::kFmtGG;+ m.K = ggt;+ m.raw = dp;+ m.k = k;+ m.n = n;+ m.ld = (int) gg::type_row_bytes(ggt, k);+ return true;+ }+ // a GGUF's BF16 or Q8_0 matrix ([out][in]) onto the device, as mat() takes an EXL3 one: whole, or part `part` of its+ // columns (outputs: a range of its rows) or of its rows (inputs: a range of each row, whole Q8_0 blocks) - a+ // standalone [n][k] matrix+ bool mat_bf16(Dev& d, const std::string& pre, const Tensor& t, x3::Mat& m, std::string& err, size_t* acct, int part,+ int np, bool cols) {+ const bool q8 = t.dtype == "Q8_0";+ // a ggml K-quant dense matrix (Q4_K / Q5_K / Q6_K: 256 values a block; the GEMV path of ds4_iq)+ const int ggt = (t.dtype == "Q4_K" || t.dtype == "Q5_K" || t.dtype == "Q6_K") ? gg_type_of(t.dtype) : -1;+ if (ggt > 0) return mat_gg(d, pre, t, ggt, m, err, acct, part, np, cols);+ if (t.shape.size() != 2 || (q8 && t.shape[1] % 32 != 0)) {+ err = pre + ": a " + t.dtype + " tensor that is not a matrix";+ return false;+ }+ int n = (int) t.shape[0], k = (int) t.shape[1];+ // a row's bytes, and the bytes of v of its values (BF16: 2 each; Q8_0: 34 every 32)+ auto bytes_of = [&](size_t v) { return q8 ? v / 32 * gg::kQ8Bytes : v * 2; };+ std::vector<uint8_t> w((size_t) n * bytes_of((size_t) k));+ if (t.bytes != w.size() || !pk.read(t, w.data())) {+ err = pre + ": read failed";+ return false;+ }+ if (np > 1) {+ const int whole = cols ? n : k, wd = whole / np;+ if (whole % np != 0 || wd % (q8 ? 32 : 8) != 0) {+ err = pre + ": " + std::to_string(whole) + (cols ? " outputs" : " inputs") + " do not split in " ++ std::to_string(np) + " parts";+ return false;+ }+ const size_t rb = bytes_of((size_t) k);+ std::vector<uint8_t> p;+ if (cols) {+ p.assign(w.begin() + (long) ((size_t) part * wd * rb), w.begin() + (long) ((size_t) (part + 1) * wd * rb));+ n = wd;+ } else {+ const size_t pb = bytes_of((size_t) wd);+ p.resize((size_t) n * pb);+ for (size_t r = 0; r < (size_t) n; ++r)+ std::memcpy(p.data() + r * pb, w.data() + r * rb + bytes_of((size_t) part * wd), pb);+ k = wd;+ }+ w.swap(p);+ }+ uint8_t* dp = nullptr;+ if (!dalloc(d, dp, w.size(), err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(dp, w.data(), w.size(), cudaMemcpyHostToDevice));+ m = x3::Mat{};+ m.fmt = q8 ? gg::kFmtQ8_0 : gg::kFmtBF16;+ m.raw = dp;+ m.k = k;+ m.n = n;+ m.ld = q8 ? (int) bytes_of((size_t) k) : k;+ return true;+ }+ // a Q8_0 tensor's values as F32 (host)+ static std::vector<float> q8_values(const uint8_t* q, size_t n) {+ std::vector<float> v(n);+ for (size_t b = 0; b < n / 32; ++b) {+ const uint8_t* p = q + b * gg::kQ8Bytes;+ const float d = f16_to_f32((uint16_t) (p[0] | p[1] << 8));+ for (int j = 0; j < 32; ++j) v[b * 32 + j] = d * (float) (int8_t) p[2 + j];+ }+ return v;+ }+ // an EXL3 matrix onto the device: whole, or (np > 1) part `part` of its columns (cols) or of its rows - a standalone+ // matrix of its own (the rotations' 128-blocks stay whole)+ // (rot: split128's - an EXL3 matrix's uneven 128-block shares going round the members; a GGUF's BF16 / Q8_0 matrix+ // splits evenly, its groups only as wide as that goes)+ bool mat(Dev& d, const std::string& pre, x3::Mat& m, std::string& err, size_t* acct, int part = 0, int np = 1,+ bool cols = true, int rot = 0) {+ if (const Tensor* tw = pk.get(pre + ".weight"); tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0" || tw->dtype == "Q4_K" || tw->dtype == "Q5_K" || tw->dtype == "Q6_K"))+ return mat_bf16(d, pre, *tw, m, err, acct, part, np, cols);+ const Tensor* tr = pk.get(pre + ".trellis");+ const Tensor* su = pk.get(pre + ".suh");+ const Tensor* sv = pk.get(pre + ".svh");+ if (tr == nullptr || su == nullptr || sv == nullptr || tr->shape.size() != 3) {+ err = pre + ": not an EXL3 matrix in the pack";+ return false;+ }+ if (pk.get(pre + ".mul1") == nullptr) {+ err = pre + ": only the mul1 codebook is covered";+ return false;+ }+ m.k = (int) tr->shape[0] * 16;+ m.n = (int) tr->shape[1] * 16;+ m.K = (int) tr->shape[2] / 16;+ m.cb = 2;+ m.ld = 0;+ if (np > 1) {+ const int whole = cols ? m.n : m.k;+ int at = 0, w = 0;+ if (whole % 128 != 0 || whole / 128 < np) {+ err = pre + ": " + std::to_string(whole) + (cols ? " columns" : " rows") + " do not split in " ++ std::to_string(np) + " parts of 128-blocks";+ return false;+ }+ split128(whole, np, part, at, w, rot);+ std::vector<uint8_t> t0(tr->bytes), u0(su->bytes), v0(sv->bytes);+ if (!pk.read(*tr, t0.data()) || !pk.read(*su, u0.data()) || !pk.read(*sv, v0.data())) {+ err = pre + ": read failed";+ return false;+ }+ const size_t tile = tr->bytes / ((size_t) (m.k / 16) * (m.n / 16));+ std::vector<uint8_t> t1, u1, v1;+ if (cols) { // the tiles [c0 / 16, (c0 + w) / 16) of each 16-row slice, svh [c0, c0 + w)+ const size_t c0 = (size_t) at, nt = (size_t) m.n / 16, nth = (size_t) w / 16;+ t1.resize((size_t) (m.k / 16) * nth * tile);+ for (size_t sl = 0; sl < (size_t) m.k / 16; ++sl)+ std::memcpy(t1.data() + sl * nth * tile, t0.data() + (sl * nt + c0 / 16) * tile, nth * tile);+ u1 = u0;+ v1.assign(v0.begin() + (long) (c0 * 2), v0.begin() + (long) ((c0 + w) * 2));+ m.n = w;+ } else { // the 16-row slices [r0 / 16, (r0 + w) / 16), suh [r0, r0 + w)+ const size_t r0 = (size_t) at, row = (size_t) (m.n / 16) * tile;+ t1.assign(t0.begin() + (long) (r0 / 16 * row), t0.begin() + (long) ((r0 + w) / 16 * row));+ u1.assign(u0.begin() + (long) (r0 * 2), u0.begin() + (long) ((r0 + w) * 2));+ v1 = v0;+ m.k = w;+ }+ uint32_t* dt = nullptr;+ uint16_t *ds = nullptr, *dv = nullptr;+ if (!dalloc(d, dt, t1.size() / 4, err, acct) || !dalloc(d, ds, u1.size() / 2, err, acct) ||+ !dalloc(d, dv, v1.size() / 2, err, acct))+ return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(dt, t1.data(), t1.size(), cudaMemcpyHostToDevice));+ CK(cudaMemcpy(ds, u1.data(), u1.size(), cudaMemcpyHostToDevice));+ CK(cudaMemcpy(dv, v1.data(), v1.size(), cudaMemcpyHostToDevice));+ m.trellis = dt;+ m.suh = ds;+ m.svh = dv;+ return x3::supported(m) || (err = pre + ": a part not covered by the EXL3 kernels", false);+ }+ uint32_t* dt = nullptr;+ uint16_t *ds = nullptr, *dv = nullptr;+ if (!dalloc(d, dt, tr->bytes / 4, err, acct) || !dalloc(d, ds, su->bytes / 2, err, acct) ||+ !dalloc(d, dv, sv->bytes / 2, err, acct))+ return false;+ if (!upload(d, *tr, dt, err) || !upload(d, *su, ds, err) || !upload(d, *sv, dv, err)) return false;+ m.trellis = dt;+ m.suh = ds;+ m.svh = dv;+ if (!x3::supported(m)) {+ err = pre + ": k " + std::to_string(m.k) + " n " + std::to_string(m.n) + " K " + std::to_string(m.K) ++ " not covered by the EXL3 kernels";+ return false;+ }+ return true;+ }+ // a float tensor (F32 / F16 / BF16) as F32 on the device+ bool f32(Dev& d, const std::string& name, float*& p, std::string& err, size_t* acct, size_t want = 0) {+ const Tensor* t = pk.get(name);+ if (t == nullptr) {+ err = name + ": not in the pack";+ return false;+ }+ std::vector<uint8_t> raw(t->bytes);+ if (!pk.read(*t, raw.data())) {+ err = name + ": read failed";+ return false;+ }+ size_t n = 0;+ std::vector<float> v;+ if (t->dtype == "F32") {+ n = t->bytes / 4;+ v.resize(n);+ std::memcpy(v.data(), raw.data(), t->bytes);+ } else if (t->dtype == "F16" || t->dtype == "BF16") {+ n = t->bytes / 2;+ v.resize(n);+ const uint16_t* s = (const uint16_t*) raw.data();+ for (size_t i = 0; i < n; ++i) v[i] = t->dtype == "F16" ? f16_to_f32(s[i]) : bf16_to_f32(s[i]);+ } else if (t->dtype == "Q8_0") { // (a GGUF's router may come so)+ n = t->bytes / gg::kQ8Bytes * 32;+ v = q8_values(raw.data(), n);+ } else {+ err = name + ": dtype " + t->dtype + " is not a float";+ return false;+ }+ if (want != 0 && n != want) {+ err = name + ": " + std::to_string(n) + " values, expected " + std::to_string(want);+ return false;+ }+ if (!dalloc(d, p, n, err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(p, v.data(), n * 4, cudaMemcpyHostToDevice));+ return true;+ }+ bool f16raw(Dev& d, const std::string& name, uint16_t*& p, std::string& err, size_t* acct) {+ const Tensor* t = pk.get(name);+ if (t != nullptr && (t->dtype == "F32" || t->dtype == "BF16" || t->dtype == "Q8_0")) { // (a GGUF's)+ const size_t n = t->dtype == "Q8_0" ? t->bytes / gg::kQ8Bytes * 32 : t->bytes / (t->dtype == "F32" ? 4 : 2);+ std::vector<uint8_t> raw(t->bytes);+ if (!pk.read(*t, raw.data())) {+ err = name + ": read failed";+ return false;+ }+ const std::vector<float> q = t->dtype == "Q8_0" ? q8_values(raw.data(), n) : std::vector<float>();+ std::vector<uint16_t> h(n);+ for (size_t i = 0; i < n; ++i) {+ float v;+ if (t->dtype == "F32") std::memcpy(&v, raw.data() + i * 4, 4);+ else if (t->dtype == "Q8_0") v = q[i];+ else v = bf16_to_f32(((const uint16_t*) raw.data())[i]);+ h[i] = strata::kernels::f16_from_f32(v);+ }+ if (!dalloc(d, p, n, err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(p, h.data(), n * 2, cudaMemcpyHostToDevice));+ return true;+ }+ if (t == nullptr || t->dtype != "F16") {+ err = name + ": not an F16 tensor in the pack";+ return false;+ }+ if (!dalloc(d, p, t->bytes / 2, err, acct)) return false;+ return upload(d, *t, p, err);+ }++ // The markov bias's head as Q8_0 blocks of 32 (34 B: F16 d + 32 int8) instead of F16 values: the same bias sums with the+ // quantisation's error, half the bytes the drafter's five markov steps read (66 MB -> 33 MB) and half the memory. The+ // pack is untouched: the F32 / F16 / BF16 tensor is quantised as it is loaded.+ bool mk_head_q8(Dev& d, const std::string& name, uint8_t*& p, std::string& err, size_t* acct) {+ const Tensor* t = pk.get(name);+ if (t == nullptr || (t->dtype != "F32" && t->dtype != "F16" && t->dtype != "BF16")) {+ err = name + ": not an F32 / F16 / BF16 tensor";+ return false;+ }+ const size_t n = t->bytes / (t->dtype == "F32" ? 4 : 2);+ if (n == 0 || n % 32 != 0) {+ err = name + ": not a whole number of 32-value blocks";+ return false;+ }+ std::vector<uint8_t> raw(t->bytes);+ if (!pk.read(*t, raw.data())) {+ err = name + ": read failed";+ return false;+ }+ std::vector<float> v(n);+ for (size_t i = 0; i < n; ++i) {+ if (t->dtype == "F32") std::memcpy(&v[i], raw.data() + i * 4, 4);+ else v[i] = t->dtype == "F16" ? f16_to_f32(((const uint16_t*) raw.data())[i])+ : bf16_to_f32(((const uint16_t*) raw.data())[i]);+ }+ const size_t nb = n / 32;+ std::vector<uint8_t> q(nb * 34);+ for (size_t b = 0; b < nb; ++b) {+ const float* s = v.data() + b * 32;+ float amax = 0.0f;+ for (int i = 0; i < 32; ++i) amax = std::max(amax, std::fabs(s[i]));+ const float dd = amax > 0.0f ? amax / 127.0f : 0.0f;+ const uint16_t dh = strata::kernels::f16_from_f32(dd);+ std::memcpy(q.data() + b * 34, &dh, 2);+ int8_t* o = (int8_t*) (q.data() + b * 34 + 2);+ for (int i = 0; i < 32; ++i) {+ const float x = dd > 0.0f ? s[i] / dd : 0.0f;+ o[i] = (int8_t) std::max(-127.0f, std::min(127.0f, std::round(x)));+ }+ }+ if (!dalloc(d, p, nb * 34, err, acct)) return false;+ cudaSetDevice(d.id);+ CK(cudaMemcpy(p, q.data(), nb * 34, cudaMemcpyHostToDevice));+ return true;+ }++ size_t mat_bytes(const std::string& pre) const {+ if (const Tensor* t = pk.get(pre + ".weight")) return align256(t->bytes);+ size_t b = 0;+ for (const char* s : {".trellis", ".suh", ".svh"})+ if (const Tensor* t = pk.get(pre + s)) b += align256(t->bytes);+ return b;+ } size_t tensor_bytes(const std::string& name) const {+ const Tensor* t = pk.get(name);+ return t ? align256(t->bytes * (t->dtype == "F32" ? 1 : 2)) : 0;+ }++ // ---- expert regions and blob layout+ // a GGUF's MXFP4 experts: the blob is gate | up | down ([out][in / 32] blocks each, as the file holds them), every+ // expert's matrix r at the same stride from expert 0's in one file (the GGUF's [n_exp][out][in] tensor)+ // (the other block types: lay.fmt stays kFmtMXFP4 - read in place as these - and lay.K[r] is role r's ggml type)+ static int gg_type_of(const std::string& dtype) {+ for (int t : {gg::kTypeMXFP4, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23, 29, 145, 146, 224, 225, 226, 227})+ if (dtype == gg::type_name(t)) return t;+ return -1;+ }+ bool expert_layout_mx(Layer& l, std::string& err) {+ const std::string P = l.name + "ffn.experts.";+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ const int n = r < 2 ? g.ff : g.dim, k = r < 2 ? g.dim : g.ff;+ const Tensor* t0 = pk.get(P + "0." + parts[r] + ".weight");+ const Tensor* t1 = l.n_exp > 1 ? pk.get(P + "1." + parts[r] + ".weight") : t0;+ const int ty = t0 != nullptr ? gg_type_of(t0->dtype) : -1;+ const size_t bytes = ty > 0 ? (size_t) n * gg::type_row_bytes(ty, k) : 0;+ if (t0 == nullptr || t1 == nullptr || bytes == 0 || t0->bytes != bytes) {+ err = P + "0." + parts[r] + ": not an MXFP4 or low-bit [" + std::to_string(n) + "][" + std::to_string(k) ++ "] matrix the engine reads (" + (t0 != nullptr ? t0->dtype : std::string("missing")) + ")";+ return false;+ }+ l.mx_file[r] = t0->file;+ l.mx_off[r] = t0->off;+ l.mx_stride[r] = l.n_exp > 1 ? t1->off - t0->off : bytes;+ for (int e = 0; e < l.n_exp; ++e) {+ const Tensor* t = pk.get(P + std::to_string(e) + "." + parts[r] + ".weight");+ if (t == nullptr || t->dtype != t0->dtype || t->bytes != bytes || t->file != l.mx_file[r] ||+ t->off != l.mx_off[r] + (uint64_t) e * l.mx_stride[r]) {+ err = P + std::to_string(e) + "." + parts[r] + ": missing, or not where the layer's others are";+ return false;+ }+ }+ l.lay.trellis[r] = off;+ l.lay.suh[r] = l.lay.svh[r] = off;+ l.lay.K[r] = ty;+ off += align256(bytes) + (store_gb() > 0 ? ExpertStore::kSlack : 0); // (the store's aligned reads spill into the gaps)+ }+ l.lay.fmt = gg::kFmtMXFP4;+ if (!gg::mx_experts(l.lay) && (gg::expert_type(l.lay, 0) != gg::expert_type(l.lay, 1) ||+ gg::expert_type(l.lay, 0) == gg::kTypeMXFP4 ||+ gg::expert_type(l.lay, 2) == gg::kTypeMXFP4)) {+ err = P + ": gate / up / down of types " + gg::type_name(l.lay.K[0]) + " / " + gg::type_name(l.lay.K[1]) ++ " / " + gg::type_name(l.lay.K[2]) + " (gate and up must match; MXFP4 does not mix with the others)";+ return false;+ }+ l.blob = off;+ // a type the device kernels do not take (none of the i-quants now: gg::type_on_device lists them): the layer stays on the CPU lane, where+ // ggml-cpu's row dots read it - one warning, then the placement code keeps it out of VRAM+ for (int r = 0; r < 3; ++r)+ if (!gg::type_on_device(l.lay.K[r])) {+ l.cpu_only = true;- std::fprintf(stderr,- "ds4: layer %d %s is %s: no GPU kernel for it, its experts stay on the CPU lane "- "(ggml-cpu's rows; VRAM tier: none)\n", l.il, r == 2 ? "down" : "gate/up",- gg::type_name(l.lay.K[r]));++ static bool said = false; // (a mixed pack has many such layers: one line, not one a layer)++ if (!said) {++ said = true;++ std::fprintf(stderr,++ "ds4: this pack has %s experts: no GPU kernel for that type, every such layer's "++ "experts stay on the CPU lane (ggml-cpu's rows; VRAM tier: none)\n",++ gg::type_name(l.lay.K[r]));++ }+ break;+ }+ return true;+ }+ const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's+ if (store) {+ const int s = store->find(l.il, e);+ if (s >= 0) return store->mat(s, r);+ }+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)+ return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]+ : ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];+ return pk.at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r]);+ }+ size_t mx_part_bytes(const Layer& l, int r) const { // (its type's)+ return r < 2 ? (size_t) g.ff * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)+ : (size_t) g.dim * gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff);+ }+ // (copy_mode) matrix r's rows, and the rows of one NUMA part of it+ int mx_n(int r) const { return r < 2 ? g.ff : g.dim; }+ size_t mx_row_b(const Layer& l, int r) const { return mx_part_bytes(l, r) / (size_t) mx_n(r); }+ // part X's rows [X n / NH, ..) of expert e's matrix r: its copy's part on node X, else where the file has them+ const uint8_t* mx_part_ptr(const Layer& l, int e, int r, int X) const {+ if (copy_mode && l.ram[(size_t) e] >= 0)+ return ram_part[(size_t) l.ram[(size_t) e]][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];+ return mx_mat(l, e, r) + (size_t) X * (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ }+ // a copy's NH parts -> the blob, and back (each matrix's rows of part X at X n / NH rows into the blob's matrix)+ void ram_to_blob(const Layer& l, int s, uint8_t* blob) const {+ for (int r = 0; r < 3; ++r) {+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ for (int X = 0; X < NH; ++X)+ std::memcpy(blob + l.lay.trellis[r] + (size_t) X * nb, ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], nb);+ }+ }+ void blob_to_ram(const Layer& l, const uint8_t* blob, int s) const {+ for (int r = 0; r < 3; ++r) {+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ for (int X = 0; X < NH; ++X)+ std::memcpy(ram_part[(size_t) s][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r], blob + l.lay.trellis[r] + (size_t) X * nb, nb);+ }+ }+ // expert e (from the file) -> its blob; and member m of nm's part of a blob (gate / up rows [m n_ff / nm, ..), down+ // inputs of the same range: in each row the blocks [m n_ff / nm / 32, ..)) and back+ bool read_mx(const Layer& l, int e, uint8_t* blob) const { // (from its RAM-tier slot or copy when it has one)+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) {+ if (copy_mode) ram_to_blob(l, l.ram[(size_t) e], blob);+ else std::memcpy(blob, ram_slots[(size_t) l.ram[(size_t) e]], l.blob);+ return true;+ }+ return read_mx_file(l, e, blob);+ }+ bool read_mx_file(const Layer& l, int e, uint8_t* blob) const { // (the movers: never a slot that may change)+ for (int r = 0; r < 3; ++r)+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) e * l.mx_stride[r], mx_part_bytes(l, r),+ blob + l.lay.trellis[r]))+ return false;+ return true;+ }+ void mx_blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {+ const int ffh = g.ff / l.nm;+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);+ for (int r = 0; r < 2; ++r)+ std::memcpy(dst + l.hlay.trellis[r], blob + l.lay.trellis[r] + (size_t) m * ffh * gr, (size_t) ffh * gr);+ for (size_t row = 0; row < (size_t) g.dim; ++row)+ std::memcpy(dst + l.hlay.trellis[2] + row * dh, blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, dh);+ }+ void mx_half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {+ const int ffh = g.ff / l.nm;+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);+ for (int r = 0; r < 2; ++r)+ std::memcpy(blob + l.lay.trellis[r] + (size_t) m * ffh * gr, src + l.hlay.trellis[r], (size_t) ffh * gr);+ for (size_t row = 0; row < (size_t) g.dim; ++row)+ std::memcpy(blob + l.lay.trellis[2] + row * dr + (size_t) m * dh, src + l.hlay.trellis[2] + row * dh, dh);+ }++ bool expert_layout(Layer& l, std::string& err) {+ if (const Tensor* t = pk.get(l.name + "ffn.experts.0.w1.weight"); t != nullptr && gg_type_of(t->dtype) > 0)+ return expert_layout_mx(l, err);+ const std::string P = l.name + "ffn.experts.";+ const char* parts[3] = {"w1", "w3", "w2"}; // gate, up, down+ const char* pieces[3] = {".trellis", ".suh", ".svh"};+ l.region_off.resize((size_t) l.n_exp);+ l.region_bytes.resize((size_t) l.n_exp);+ l.region_file.resize((size_t) l.n_exp);+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ const Tensor* tr = pk.get(P + "0." + parts[r] + ".trellis");+ if (tr == nullptr || pk.get(P + "0." + parts[r] + ".mul1") == nullptr) {+ err = P + "0." + parts[r] + ": not an EXL3 (mul1) matrix";+ return false;+ }+ l.lay.K[r] = (int) tr->shape[2] / 16;+ l.lay.trellis[r] = off;+ off += align256(tr->bytes);+ l.lay.suh[r] = off;+ off += align256(pk.get(P + "0." + parts[r] + ".suh")->bytes);+ l.lay.svh[r] = off;+ off += align256(pk.get(P + "0." + parts[r] + ".svh")->bytes);+ }+ l.lay.cb = 2;+ l.blob = off;+ for (int e = 0; e < l.n_exp; ++e) {+ const std::string E = P + std::to_string(e) + ".";+ uint64_t lo = UINT64_MAX, hi = 0;+ int file = -1;+ for (int r = 0; r < 3; ++r)+ for (int q = 0; q < 3; ++q) {+ const Tensor* t = pk.get(E + parts[r] + pieces[q]);+ if (t == nullptr) {+ err = E + parts[r] + pieces[q] + ": missing";+ return false;+ }+ if (file < 0) file = t->file;+ if (t->file != file) {+ err = E + ": split across files";+ return false;+ }+ lo = std::min(lo, t->off);+ hi = std::max(hi, t->off + t->bytes);+ if (e == 0) l.piece_bytes[r][q] = t->bytes;+ else if (t->bytes != l.piece_bytes[r][q]) {+ err = E + parts[r] + ": not the shape of expert 0's";+ return false;+ }+ }+ l.region_off[(size_t) e] = lo;+ l.region_bytes[(size_t) e] = hi - lo;+ l.region_file[(size_t) e] = file;+ for (int r = 0; r < 3; ++r)+ for (int q = 0; q < 3; ++q) {+ const size_t src = pk.get(E + parts[r] + pieces[q])->off - lo;+ if (e == 0) l.piece_src[r][q] = src;+ else if (src != l.piece_src[r][q]) {+ err = E + ": the pieces lie differently from expert 0's";+ return false;+ }+ }+ }+ return true;+ }+ // the region of expert e (read from the file) -> its blob+ void repack(const Layer& l, const uint8_t* region, uint8_t* blob) const {+ for (int r = 0; r < 3; ++r) {+ const size_t dst[3] = {l.lay.trellis[r], l.lay.suh[r], l.lay.svh[r]};+ for (int q = 0; q < 3; ++q) std::memcpy(blob + dst[q], region + l.piece_src[r][q], l.piece_bytes[r][q]);+ }+ }++ // the region of expert e -> its CPU-lane part X (columns [X n / NH, (X + 1) n / NH) of each matrix)+ void repack_part(const Layer& l, const uint8_t* region, int X, uint8_t* dst) const {+ for (int r = 0; r < 3; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;+ const uint8_t* tr = region + l.piece_src[r][0];+ for (int sl = 0; sl < k / 16; ++sl)+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,+ (size_t) nth * tile);+ std::memcpy(dst + l.hl.suh[r], region + l.piece_src[r][1], (size_t) k * 2);+ std::memcpy(dst + l.hl.svh[r], region + l.piece_src[r][2] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);+ }+ }++ // ---- loading+ bool load(const Options& opt, std::string& err) {+ o = opt;+ const double t0 = now_ms();+ if (!pk.load(o.pack, err)) return false;+ g.from(pk);+ if (g.dim == 0 || g.nl == 0 || (int) g.ratios.size() < g.nl) {+ err = o.pack + "/ds4.txt: incomplete geometry";+ return false;+ }+ if (g.hd != 512 || g.ix_hd != 128 || g.k_exp > 8 || g.hc > 8) {+ err = "geometry not covered (head_dim " + std::to_string(g.hd) + ", index_head_dim " ++ std::to_string(g.ix_hd) + ", experts per token " + std::to_string(g.k_exp) + ")";+ return false;+ }+ freqs_win = rope_freqs(g.rd, 0, g.rope_theta, g.factor, g.beta_fast, g.beta_slow);+ freqs_comp = rope_freqs(g.rd, g.orig_ctx, g.comp_theta, g.factor, g.beta_fast, g.beta_slow);++ // devices+ int ndev = 0;+ if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev == 0) {+ err = "no CUDA device";+ return false;+ }+ std::vector<int> ids = o.gpus;+ if (ids.empty())+ for (int i = 0; i < ndev; ++i) ids.push_back(i);+ devs.resize(ids.size());+ for (size_t i = 0; i < ids.size(); ++i) {+ Dev& d = devs[i];+ d.id = ids[i];+ CK(cudaSetDevice(d.id));+ CK(cudaStreamCreateWithFlags(&d.st, cudaStreamNonBlocking));+ CK(cudaEventCreateWithFlags(&d.ev, cudaEventDisableTiming));++ CK(cudaEventCreateWithFlags(&d.ev2, cudaEventDisableTiming));+ if (!x3::prepare(d.st) || (pk.gguf && !gg::prepare(d.st))) {+ err = "exl3 scratch on CUDA" + std::to_string(d.id);+ return false;+ }+ CK(cudaMemGetInfo(&d.free0, &d.total));+ }+ // the groups' width: o.tp, less while the experts' blocks do not split in it (a member takes n_ff / tp of each+ // expert: the down's inputs in whole blocks - a GGUF's 256-value types split n_ff 2304 in 1, 3 or 9 parts)+ int tp = o.tp;+ if (pk.gguf && tp > 1) {+ int blk = 32;+ for (const auto& kv : pk.t) {+ const std::string& n = kv.first;+ if (n.find(".ffn.experts.") == std::string::npos || n.size() < 9 ||+ n.compare(n.size() - 9, 9, "w2.weight") != 0)+ continue;+ const int ty = gg_type_of(kv.second.dtype);+ if (ty >= 0) blk = std::max(blk, gg::type_block(ty));+ }+ while (tp > 1 && (g.ff % tp != 0 || (g.ff / tp) % blk != 0 || (g.ff / tp) % (std::getenv("DS4_TP_ALIGN") ? std::atoi(std::getenv("DS4_TP_ALIGN")) : 128) != 0 || g.heads % tp != 0 ||+ g.o_groups % tp != 0))+ --tp;+ if (tp != o.tp)+ std::fprintf(stderr, "ds4: tensor parallel %d -> %d: the experts' %d-value blocks split n_ff %d no other way "+ "(each GPU its own layers)\n", o.tp, tp, blk, g.ff);+ }+ // the groups: runs of tp devices of one kind (their memory), a device unlike the next alone+ // - or, tp the number of GPUs, one group of all of them whatever their kind (its tightest member sets the+ // experts' room); threads mode for the windows (decode) where a group is 2 wide or more or --threads on, the+ // prompt path (its own thread a GPU) as asked for - also where the experts' blocks took the groups' width down+ // to 1 (a 256-value type's: Maya's quants), which left every prompt in windows of 16+ par = o.threads || tp > 1;+ // the chunked (layer-major) prompt path: with --tp / --threads as before; --prefill-chunk N (N > 0) or+ // kChunkedPromptDefault bring it to a single card's default run too (a host thread a GPU for the prompt only;+ // decode stays as asked). --prefill-chunk 0 keeps the windows of 16 (DS4_WINDOWS=1 the same). Off by default+ // until a prompt costs the decode nothing: its buffers hold VRAM the windowed path gives to experts+ prompt_par = par || o.tp > 1 ||+ ((o.prefill_chunk > 0 || (o.prefill_chunk < 0 && kChunkedPromptDefault)) &&+ std::getenv("DS4_WINDOWS") == nullptr);++#if defined(STRATA_METAL)++ if (std::getenv("DS4_NO_CHUNKS") == nullptr) prompt_par = true; // the chunked prompt path on one device, the decode steps stay on the host-routed path++#endif+ const bool tp_all = tp > 1 && tp == (int) devs.size();+ for (size_t i = 0; i < devs.size();) {+ Group G;+ G.mem.push_back((int) i);+ size_t j = i + 1;+ while ((int) G.mem.size() < tp && j < devs.size() && (tp_all || devs[j].total == devs[i].total))+ G.mem.push_back((int) j++);+ if ((int) G.mem.size() < tp && G.mem.size() > 1) { // a short run at the end: devices of their own+ for (int m : G.mem) {+ Group one;+ one.mem.push_back(m);+ groups.push_back(one);+ }+ } else groups.push_back(G);+ i = j;+ }+ for (size_t gi = 0; gi < groups.size(); ++gi)+ for (size_t m = 0; m < groups[gi].mem.size(); ++m) {+ devs[(size_t) groups[gi].mem[m]].grp = (int) gi;+ devs[(size_t) groups[gi].mem[m]].mi = (int) m;+ }+ // whether the CPU lane's experts are read in place (--mmap-experts; auto: the pack's expert bytes beyond what the+ // GPUs can take - their free memory less the other weights and the prompt path's buffers - do not fit the RAM+ // this process can get beside kRamMargin), decided here because it sets the prompt chunk's default: in place+ // every chunk streams every expert VRAM does not hold from the files, so the larger the chunk the fewer times+ if (pk.gguf) file_experts = true; // (a GGUF's: the files stay their source - copy_experts copies them into RAM+ // after VRAM's are in, by --mmap-experts)+ else if (o.mmap_experts >= 0) file_experts = o.mmap_experts > 0;+ else {+ double exp_b = 0, other_b = 0, room = 0;+ for (const auto& kt : pk.t) {+ const std::string& n = kt.first;+ if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||+ (!o.dspark && n.rfind("mtp.", 0) == 0))+ continue;+ (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;+ }+ for (const Dev& d : devs)+ room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) -+ (prompt_par ? 1.5 : 0.4) * (1 << 30)) *+ o.vram_experts;+ file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();+ }+ // The default prompt chunk, one rule (ds4_experts_rule.hpp: prefill_chunk_default). Measured: the chunked pass streams the experts the GPUs+ // do not hold once per chunk, so where they hold less than half of them a larger chunk is fewer passes (1 x R9700 + 251 GB, 8K prompt:+ // 1024 200, 2048 277, 4096 353, 8192 431 tok/s; RTX 5070 / 3060 12 GB in place: 8192 244 / 228 against 1024 186 / 175); where they hold+ // most (4 x R9700: 8192 214, 1024 370) the small one wins; with the experts copied into RAM (--mmap-experts off) 4096 beats 8192 by 7-9%+ // (H100 32 GiB: 2048 502, 4096 538, 8192 495). A GGUF pack always has file_experts set (copied only by --mmap-experts off).+ const bool own_default = !(par || o.tp > 1); // (a run with tp 1 and no --threads: `strata launch` writes --tp 1 for 4+ AMD cards, --tp 2 for 2-3 and nothing for one card = the engine's tp 2)+ const bool multi_tp = rdna4() && !own_default && !o.threads && devs.size() >= 2; // (the 4-card default-tp run the 1024 chunk was also measured on; an explicit --prefill-chunk below wins over the rule)+ const bool gguf_copy = pk.gguf && (o.mmap_experts == 0 || (store_gb() <= 0.0 && auto_copy_all));+ uint64_t vmin_bytes = ~0ull;+ for (const Dev& dv : devs) vmin_bytes = std::min<uint64_t>(vmin_bytes, (uint64_t) dv.total);+ if (devs.empty()) vmin_bytes = 0;+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)+ const bool hip_build = true;+#else+ const bool hip_build = false;+#endif+ const int rule_chunk = prefill_chunk_default(own_default, gpu_expert_share(), gguf_copy, file_experts, hip_build, vmin_bytes, kVlRows, multi_tp, defaults_026());+ chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : rule_chunk) : 0;+ if (prompt_par && o.prefill_chunk < 0 && rule_chunk == 4096 && gguf_copy)+ std::fprintf(stderr, "ds4: prompts in chunks of 4096 tokens (the experts are copied into RAM, %.0f GiB of graphics memory; --prefill-chunk N changes it)\n",+ (double) vmin_bytes / (double) (1ull << 30));+ const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)+ if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;+ if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));+ if (chunk > 0) {+ chunk = std::min(kMaxChunk, std::max(dc != nullptr ? 16 : 256, chunk / 16 * 16));+ eg_cap = (size_t) chunk;+ tap_cap = std::max(kWin, 128);+ } else // (only as asked for: --prefill-chunk 0, or --tp 1 with --threads off)+ std::fprintf(stderr,+ "ds4: prompts run in windows of %d tokens (%s): long prompts prefill far slower than in\n"+ " chunks (a 7-GPU RTX 3090 rig, Maya-S read in place: 8K / 32K tokens 115 / 99 tokens/s,\n"+ " 461 / 1010 in chunks of %d) - %s\n",+ kWin, o.prefill_chunk == 0 ? "--prefill-chunk 0" : "--tp 1 with --threads off", kVlRows,+ o.prefill_chunk == 0 ? "leave --prefill-chunk out for chunks"+ : "leave --tp 1 out, or pass --threads on, for chunks");+ // the vision tower on the first GPU, before the split: the layers get what it leaves+ if (o.vision && !load_vision(err)) { // (a tower that cannot load leaves the model reading text: said loudly,+ if (o.vision_only) return false; // and the server is told pictures are off - info_lines)+ std::fprintf(stderr, "ds4: WARNING: the vision tower did not load (%s): pictures are OFF, the model reads "+ "text only\n", err.c_str());+ vision_err = err.empty() ? "unknown reason" : err;+ err.clear();+ vt.release();+ img_tok = -1;+ }+ if (o.vision_only) return true;++ // layers: roles and the sources they read+ if (o.dspark_auto && o.dspark && g.n_mtp > 0) { // (see Options::dspark_auto)+ double exp_b = 0, room_b = 0;+ for (const auto& kt : pk.t)+ if (kt.first.rfind("layers.", 0) == 0 && kt.first.find(".ffn.experts.") != std::string::npos) exp_b += (double) kt.second.bytes;+ for (const Dev& d : devs) room_b += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20));+ if (exp_b > 1.5 * room_b) {+ o.dspark = false;+ std::fprintf(stderr, "ds4: DSpark not loaded: the routed experts (%.0f GB) are far more than the GPU holds (%.0f GB), where its verify "+ "rounds read 3.3x a token's bytes and gain nothing (--dspark on loads it anyway)\n", exp_b / 1e9, room_b / 1e9);+ }+ }+ if (!o.dspark) g.n_mtp = 0;+ if (o.dspark_block > 0 && std::getenv("DS4_DSPARK_MAX") == nullptr) max_drafts = o.dspark_block; // (--dspark-block N: at most N drafts a round)+ L.resize((size_t) (g.nl + g.n_mtp));+ int last_src = -1;+ int experts_used = 1 << 20;+ for (int il = 0; il < g.nl + g.n_mtp; ++il) {+ Layer& l = L[(size_t) il];+ l.il = il;+ l.mtp = il >= g.nl;+ l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";+ l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;+ l.k_exp = l.mtp ? g.d_k : g.k_exp;+ if (!l.mtp) { // experts a word takes: --experts N / the faster-mode rule (ds4_experts_rule.hpp), DS4_TOPK=N over both (a test)+ if (il == 0) {+ uint64_t vram_all = 0;+ bool integrated = false; // (a GPU that shares the PC's RAM: Strix Halo)+ for (const Dev& dv : devs) {+ vram_all += (uint64_t) dv.total;+ cudaDeviceProp ip{};+ if (cudaGetDeviceProperties(&ip, dv.id) == cudaSuccess && ip.integrated) integrated = true;+ }+ // two or more RDNA4 cards holding at least half of the experts (the 4 x R9700 class): Faster by default too+ const bool multi_rdna4 = rdna4() && devs.size() >= 2 && gpu_expert_share() >= 0.5;+ const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp, integrated, multi_rdna4, defaults_026()), l.k_exp);+ experts_used = n;+ if (n < l.k_exp)+ std::fprintf(stderr, "ds4: experts per word: %d (%s; full quality: --experts %d)\n", n,+ o.experts >= 1 ? "as asked" : multi_rdna4 ? "faster mode: the cards hold most of the experts" : "faster mode", l.k_exp);+ else if (o.experts >= 1)+ std::fprintf(stderr, "ds4: experts per word: %d (full quality)\n", n);+ }+ l.k_exp = std::min(l.k_exp, experts_used);+ }+ l.ratio = g.ratios[(size_t) il];+ if (l.mtp) { // window attention only: no compressor, indexer or Engram+ if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {+ err = l.name + ": a DSpark block of compress ratio " + std::to_string(l.ratio) + ", " ++ std::to_string(l.n_exp) + " experts / " + std::to_string(l.k_exp) + " is not covered";+ return false;+ }+ if (!expert_layout(l, err)) return false;+ continue;+ }+ l.kv_src = std::find(g.kv_src.begin(), g.kv_src.end(), il) != g.kv_src.end();+ l.ix_src = std::find(g.ix_src.begin(), g.ix_src.end(), il) != g.ix_src.end();+ l.owns_k = l.kv_src;+ l.cand_src = il == g.cand_src;+ l.uses_cand = g.cand_src >= 0 && g.cand_src < il && l.ix_src;+ if (l.kv_src) {+ Src s;+ s.layer = il;+ s.ratio = l.ratio;+ s.cap = (o.max_context + l.ratio - 1) / l.ratio + 1;+ srcs.push_back(s);+ last_src = (int) srcs.size() - 1;+ }+ if (l.ratio > 0) {+ if (last_src < 0 || srcs[(size_t) last_src].ratio != l.ratio) {+ err = "layer " + std::to_string(il) + ": compress ratio " + std::to_string(l.ratio) ++ " without a KV source of that ratio before it";+ return false;+ }+ l.src = last_src;+ l.ik_src = last_src;+ }+ auto eg = std::find(g.eg_layers.begin(), g.eg_layers.end(), il);+ if (eg != g.eg_layers.end()) l.eg = (int) (eg - g.eg_layers.begin());+ if (!expert_layout(l, err)) return false;+ }+++ // A layer whose experts the chunk path's tile GEMM cannot take (IQ1_M on Metal today) would be streamed to the++ // GPU for every prompt and stop in "kernel launch with no Metal dispatch". Prompts then run in windows of 16,++ // which is slower but correct; say so instead of failing at the first prompt.++ {++ bool chunk_unsupported = false;++ const char* why = "";++ for (const Layer& l : L) {++ for (int r = 0; r < 3; ++r)++ if (!gg::type_on_chunk(gg::expert_type(l.lay, r))) { chunk_unsupported = true; why = gg::type_name(gg::expert_type(l.lay, r)); break; }++ if (chunk_unsupported) break;++ }++ if (chunk > 0 && chunk_unsupported) {++ chunk = 0;++ std::fprintf(stderr,++ "ds4: the prompt chunk path has no GPU kernel for %s: prompts run in windows of %d instead "++ "of chunks - slower prompts, and the only correct path on this build\n",++ why, kWin);++ }++ }+++ // the split: each layer's dense weights + its experts, the head on the last GPU+ for (Layer& l : L) {+ const std::string& P = l.name;+ size_t b = 0;+ if (l.mtp && l.il == g.nl) b += mat_bytes("mtp.0.main_proj");+ for (const char* m : {"attn.wq_a", "attn.wq_b", "attn.wkv", "attn.wo_b", "ffn.shared_experts.w1",+ "ffn.shared_experts.w2", "ffn.shared_experts.w3", "attn.compressor.wkv",+ "attn.compressor.wgate", "attn.indexer.wq_b", "attn.indexer.wk", "engram.wkv"})+ b += mat_bytes(P + m);+ for (int gi = 0; gi < g.o_groups; ++gi) b += mat_bytes(P + "attn.wo_a.slice." + std::to_string(gi));+ b += tensor_bytes(P + "hc_attn_fn") + tensor_bytes(P + "hc_ffn_fn") + tensor_bytes(P + "ffn.gate.weight");+ b += (size_t) g.win * win_rb() + (2u << 20);+ l.dense_bytes = b;+ }+ const size_t head_b = mat_bytes("head") + (size_t) g.vocab * 4 + (16u << 20);+ const size_t per_dev_fixed = (size_t) o.reserve_mib << 20;+ const size_t kv_per_src_dev = 0; // counted after the split+ (void) kv_per_src_dev;+ std::vector<double> avail(groups.size(), 0.0);+ double avail_sum = 0;+ for (size_t gi = 0; gi < groups.size(); ++gi) {+ for (int m : groups[gi].mem)+ avail[gi] += std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed - 400.0 * (1 << 20));+ avail_sum += avail[gi];+ }+ double need_sum = (double) head_b;+ for (const Layer& l : L) need_sum += (double) l.dense_bytes + (double) l.blob * l.n_exp;+ {+ // contiguous layer ranges, each group's share of the total by its free memory+ size_t d = 0;+ double cum = 0, target = avail[0] / avail_sum * need_sum;+ std::vector<Layer*> order; // the drafter's blocks first (the first GPU), then the trunk's layers+ for (Layer& l : L)+ if (l.mtp) order.push_back(&l);+ for (Layer& l : L)+ if (!l.mtp) order.push_back(&l);+ for (Layer* lp : order) {+ Layer& l = *lp;+ const double lb = (double) l.dense_bytes + (double) l.blob * l.n_exp;+ if (cum + lb / 2 > target && d + 1 < groups.size()) {+ ++d;+ target += avail[d] / avail_sum * need_sum;+ }+ l.grp = (int) d;+ cum += lb;+ }+ // with the chunked prompt path: the contiguous split that streams the least into its busiest member (its+ // VRAM short of its layers' experts, per member) - a chunk's groups run as a pipeline, its slowest stage+ // sets the pace; a token's decode passes every layer either way, the experts in VRAM the same in all+ if (chunk > 0 && groups.size() > 1 && groups.size() <= 6 && std::getenv("DS4_SPLIT") == nullptr) {+ const int G = (int) groups.size(), NL = g.nl;+ std::vector<double> cap(groups.size());+ for (int gi = 0; gi < G; ++gi) {+ const int nm = (int) groups[(size_t) gi].mem.size();+ double mn = 1e300;+ for (int m : groups[(size_t) gi].mem)+ mn = std::min(mn, std::max(0.0, (double) devs[(size_t) m].free0 - (double) per_dev_fixed -+ 400.0 * (1 << 20) - 1.1 * (1 << 30))); // (the chunk's buffers)+ cap[(size_t) gi] = mn * nm;+ }+ for (const Layer& l : L)+ if (l.mtp) cap[0] -= (double) l.dense_bytes + (double) l.blob * l.n_exp;+ cap[(size_t) G - 1] -= (double) head_b;+ std::vector<double> pe(NL + 1, 0.0), pd(NL + 1, 0.0); // prefix sums: experts, dense+ for (int il = 0; il < NL; ++il) {+ pe[(size_t) il + 1] = pe[(size_t) il] + (double) L[(size_t) il].blob * L[(size_t) il].n_exp;+ pd[(size_t) il + 1] = pd[(size_t) il] + (double) L[(size_t) il].dense_bytes;+ }+ auto cost = [&](int gi, int a, int b) { // per-member streaming of layers [a, b) on group gi+ const int nm = (int) groups[(size_t) gi].mem.size();+ const double dense = (pd[(size_t) b] - pd[(size_t) a]) * (nm > 1 ? 1.3 : 1.0);+ return std::max(0.0, pe[(size_t) b] - pe[(size_t) a] - (cap[(size_t) gi] - dense)) / nm;+ };+ // best[gi][b]: the least max cost of layers [0, b) over groups 0..gi (each at least one layer)+ std::vector<std::vector<double>> best((size_t) G, std::vector<double>((size_t) NL + 1, 1e300));+ std::vector<std::vector<int>> cut((size_t) G, std::vector<int>((size_t) NL + 1, 0));+ for (int b = 1; b <= NL; ++b) best[0][(size_t) b] = cost(0, 0, b);+ for (int gi = 1; gi < G; ++gi)+ for (int b = gi + 1; b <= NL; ++b)+ for (int a = gi; a < b; ++a) {+ const double v = std::max(best[(size_t) gi - 1][(size_t) a], cost(gi, a, b));+ if (v < best[(size_t) gi][(size_t) b]) {+ best[(size_t) gi][(size_t) b] = v;+ cut[(size_t) gi][(size_t) b] = a;+ }+ }+ int b = NL;+ for (int gi = G - 1; gi >= 0; --gi) {+ const int a = gi > 0 ? cut[(size_t) gi][(size_t) b] : 0;+ for (int il = a; il < b; ++il) L[(size_t) il].grp = gi;+ b = a;+ }+ }+ // DS4_SPLIT=a,b,..: the trunk layers each group takes (the drafter stays with the first)+ if (const char* sp = std::getenv("DS4_SPLIT")) {+ std::vector<int> n;+ for (const char* q = sp; *q;) {+ n.push_back(std::atoi(q));+ while (*q && *q != ',') ++q;+ if (*q == ',') ++q;+ }+ int il = 0;+ for (size_t gi = 0; gi < n.size() && gi < groups.size(); ++gi)+ for (int j = 0; j < n[gi] && il < g.nl; ++j) L[(size_t) il++].grp = (int) gi;+ for (; il < g.nl; ++il) L[(size_t) il].grp = (int) groups.size() - 1;+ }+ for (Layer& l : L) {+ l.dev = groups[(size_t) l.grp].mem[0];+ l.nm = l.mtp ? 1 : (int) groups[(size_t) l.grp].mem.size(); // (the drafter's blocks: on the lead alone)+ }+ head_dev = L[(size_t) g.nl - 1].dev;+ d_dev = g.n_mtp > 0 ? L[(size_t) g.nl].dev : 0;+ }+ // the groups' members: copies of each layer's device state, a member's part of an expert+ for (Layer& l : L) {+ l.ff0 = 0;+ l.ffn = g.ff;+ l.ah0 = l.og0 = 0;+ l.ahn = g.heads;+ l.ogn = g.o_groups;+ if (l.nm == 1) continue;+ const int na = head_members(l.nm);+ if (g.heads % na != 0 || g.o_groups % na != 0 || g.ff % 128 != 0 || g.ff / 128 < l.nm) {+ err = "tensor parallel " + std::to_string(l.nm) + ": the heads, output groups or FFN do not split";+ return false;+ }+ for (int m = 1; m < l.nm; ++m) {+ Layer c = l;+ c.mates.clear();+ c.mi = m;+ c.dev = groups[(size_t) l.grp].mem[(size_t) m];+ l.mates.push_back(std::move(c));+ }+ // each member's FFN columns and its part of an expert's layout: an EXL3 expert's in whole 128-blocks, as even+ // as they go (split128, the odd blocks going round the members layer by layer); a GGUF's in even shares (its+ // groups are only as wide as its experts' blocks split evenly: mx_blob_to_half's n_ff / nm)+ const bool gguf_x = l.lay.fmt == gg::kFmtMXFP4;+ for (int m = 0; m < l.nm; ++m) {+ Layer& x = member(l, m);+ const int ha = m - (l.nm - na); // (its place among the members with heads)+ x.ahn = ha < 0 ? 0 : g.heads / na;+ x.ah0 = ha < 0 ? 0 : ha * x.ahn;+ x.ogn = ha < 0 ? 0 : g.o_groups / na;+ x.og0 = ha < 0 ? 0 : ha * x.ogn;+ if (gguf_x) {+ x.ffn = g.ff / l.nm;+ x.ff0 = m * x.ffn;+ } else+ split128(g.ff, l.nm, m, x.ff0, x.ffn, l.il % l.nm); // (rot: the odd blocks go round the members)+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ const int k = r < 2 ? g.dim : x.ffn, n = r < 2 ? x.ffn : g.dim;+ x.hlay.K[r] = l.lay.K[r];+ x.hlay.trellis[r] = off;+ if (gguf_x) { // gate / up rows, down inputs: [n][k / block] blocks+ const int ty = gg::expert_type(l.lay, r);+ if (k % gg::type_block(ty) != 0) {+ err = "tensor parallel " + std::to_string(l.nm) + ": the experts' " + gg::type_name(ty) ++ " blocks do not split";+ return false;+ }+ x.hlay.suh[r] = x.hlay.svh[r] = off;+ off += align256((size_t) n * gg::type_row_bytes(ty, k));+ continue;+ }+ off += align256(x3::trellis_bytes(k, n, l.lay.K[r]));+ x.hlay.suh[r] = off;+ off += align256((size_t) k * 2);+ x.hlay.svh[r] = off;+ off += align256((size_t) n * 2);+ }+ x.hlay.cb = l.lay.cb;+ x.hlay.fmt = l.lay.fmt;+ x.hblob = off;+ }+ }+ // which devices hold each source's compressed KV / index keys+ for (Src& s : srcs) {+ s.ckv.assign(devs.size(), nullptr);+ s.ik.assign(devs.size(), nullptr);+ }+ for (const Layer& l0 : L)+ for (int m = 0; m < l0.nm; ++m) {+ const Layer& l = member(l0, m);+ if (l.src >= 0) {+ srcs[(size_t) l.src].ckv[(size_t) l.dev] = (uint8_t*) 1;+ if (l0.grp != L[(size_t) srcs[(size_t) l.src].layer].grp) srcs[(size_t) l.src].remote = true;+ }+ if (l.ix_src && l.ik_src >= 0) {+ srcs[(size_t) l.ik_src].ik[(size_t) l.dev] = (uint8_t*) 1;+ if (l0.grp != L[(size_t) srcs[(size_t) l.ik_src].layer].grp) srcs[(size_t) l.ik_src].remote = true;+ }+ }++ // per device: the scratch, the caches, the dense weights+ for (size_t di = 0; di < devs.size(); ++di) {+ Dev& d = devs[di];+ const int D = g.dim, H = g.hc;+ const size_t W = kWin, P = (size_t) kWin * kPicks, cap = (size_t) o.max_context + 64;+ bool ok = dalloc(d, d.h, W * H * D, err) && dalloc(d, d.h2, W * H * D, err) && dalloc(d, d.x, W * D, err) &&+ dalloc(d, d.qr, W * g.q_lora, err) && dalloc(d, d.q, W * g.heads * g.hd, err) &&+ dalloc(d, d.kvr, W * g.hd, err) && dalloc(d, d.ckr, W * g.hd, err) &&+ dalloc(d, d.cgr, W * g.hd, err) && dalloc(d, d.latent, W * g.hd, err) &&+ dalloc(d, d.ikr, W * g.ix_hd, err) && dalloc(d, d.iq, W * g.ix_heads * g.ix_hd, err) &&+ dalloc(d, d.iw, W * 64, err) && dalloc(d, d.scores, W * cap, err) &&+ dalloc(d, d.bm, W * bm_stride(), err) && dalloc(d, d.o, W * g.heads * g.hd, err) &&+ dalloc(d, d.oa, W * g.o_groups * g.o_lora, err) && dalloc(d, d.ao, W * D, err) &&+ dalloc(d, d.mix, W * 32, err) && dalloc(d, d.a_pre, W * 8, err) && dalloc(d, d.a_post, W * 8, err) &&+ dalloc(d, d.a_comb, W * 64, err) && dalloc(d, d.f_pre, W * 8, err) &&+ dalloc(d, d.f_post, W * 8, err) && dalloc(d, d.f_comb, W * 64, err) && dalloc(d, d.pre, W * 8, err) &&+ dalloc(d, d.glog, W * 1024, err) && dalloc(d, d.glog_a, 2 * W * 1024, err) &&+ dalloc(d, d.rw, W * kPicks, err) &&- dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.moe_out, W * D, err) &&++ dalloc(d, d.ridx, W * kPicks, err) && dalloc(d, d.ridx_a, 2 * kPicks, err) && dalloc(d, d.rw_a, 2 * kPicks, err) &&++ dalloc(d, d.moe_out, W * D, err) &&+ dalloc(d, d.sg, W * g.ff, err) && dalloc(d, d.su, W * g.ff, err) && dalloc(d, d.sh, W * g.ff, err) &&+ dalloc(d, d.sh_out, W * D, err) && dalloc(d, d.cpu_part, W * D, err) &&+ dalloc(d, d.freqs_win, 64, err) && dalloc(d, d.freqs_comp, 64, err) &&+ dalloc(d, d.topk, W * topk_stride(), err) && dalloc(d, d.cand, W * cand_stride(), err) &&+ dalloc(d, d.tok, 16, err) && dalloc(d, d.emb_rows, W * D, err) &&+ dalloc(d, d.ext, ((size_t) g.win + W) * g.hd, err) && dalloc(d, d.xe, P * D, err) &&+ dalloc(d, d.xg, P * g.ff, err) && dalloc(d, d.xu, P * g.ff, err) && dalloc(d, d.xhh, P * g.ff, err) &&+ dalloc(d, d.ed, P * D, err) && dalloc(d, d.plan_ptr, P, err);++ if (ok && std::getenv("DS4_DBG_DEVH"))++ std::fprintf(stderr, "DBG devh: id=%d h=%p h2=%p\n", d.id, (void*) d.h, (void*) d.h2);+ if (!ok) return false;+ d.attn_scr.max_t = kAttnScratchT;+ d.attn_scr.cap = (int) topk_stride();+ if (!dalloc(d, d.attn_scr.rows, (size_t) kAttnScratchT * topk_stride() * g.hd, err)) return false;+ if (!g.eg_layers.empty()) {+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;+ ok = dalloc(d, d.eg_rows, W * cols * (pk.eg_hd + pk.eg_hd / 32), err) &&+ dalloc(d, d.eg_x, W * cols * pk.eg_hd, err) && dalloc(d, d.eg_kv, W * (g.hc + 1) * D, err);+ if (!ok) return false;+ }+ if ((int) di == head_dev && !dalloc(d, d.logits, (size_t) kWin * g.vocab, err)) return false;+ if (!halloc(d.bounce, bounce_bytes(), err)) return false;+ if (!dalloc(d, d.xcnt, 4, err) || !dalloc(d, d.hccnt, kWin, err) || !hmap(d.hand, bounce_bytes(), err) || !hmap(d.hand_flag, 32, err) ||+ !dalloc(d, d.grp_ptr, P, err) || !dalloc(d, d.grp_start, P + 1, err) || !dalloc(d, d.grp_ent, P, err) ||+ !dalloc(d, d.n_groups, 4, err))+ return false;+ if (g.n_mtp > 0 && !dalloc(d, d.tap, (size_t) tap_cap * D, err)) return false;+ if (g.n_mtp > 0 && (int) di == head_dev &&+ (!dalloc(d, d_confv, kWin, err) || !dalloc(d, d_ints, 64, err) || !dalloc(d, d_specq, kWin, err)))+ return false;+ CK(cudaEventCreateWithFlags(&d.ev_out, cudaEventDisableTiming));+ cudaSetDevice(d.id);+ CK(cudaMemcpy(d.freqs_win, freqs_win.data(), freqs_win.size() * 4, cudaMemcpyHostToDevice));+ CK(cudaMemcpy(d.freqs_comp, freqs_comp.data(), freqs_comp.size() * 4, cudaMemcpyHostToDevice));+ for (Src& s : srcs) {+ if (s.h_ckv == nullptr) { // (its events belong to the source layer's device: it records them)+ if (!halloc(s.h_ckv, (size_t) kWin * ckv_rb(), err) || !halloc(s.h_ik, (size_t) kWin * ik_rb(), err))+ return false;+ CK(cudaSetDevice(devs[(size_t) L[(size_t) s.layer].dev].id));+ CK(cudaEventCreateWithFlags(&s.ev_ckv, cudaEventDisableTiming));+ CK(cudaEventCreateWithFlags(&s.ev_ik, cudaEventDisableTiming));+ CK(cudaSetDevice(d.id));+ }+ if (s.ckv[di] != nullptr && !dalloc(d, s.ckv[di], (size_t) s.cap * ckv_rb(), err, &d.caches))+ return false;+ if (s.ik[di] != nullptr && !dalloc(d, s.ik[di], (size_t) s.cap * ik_rb(), err, &d.caches))+ return false;+ }+ }+ for (Src& sr : srcs) {+ if (sr.remote && (!hmap(sr.x_stage, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag, 32, err)))+ return false;+ if (sr.remote && chunk > 0 &&+ (!hmap(sr.x_stage1, x_cap() * (ckv_rb() + ik_rb()), err) || !hmap(sr.x_flag1, 32, err) ||+ !hmap(sr.x_ack, (size_t) 2 * devs.size() * 32, err)))+ return false;+ const int sg = L[(size_t) sr.layer].grp;+ for (size_t di = 0; di < devs.size(); ++di)+ if ((sr.ckv[di] != nullptr || sr.ik[di] != nullptr) && devs[di].grp != sg) sr.readers.push_back((int) di);+ }+ for (Group& G : groups) {+ G.ar.nm = (int) G.mem.size();+ G.ar.slots = 4;+ G.ar.stride = (size_t) kWin * g.dim;+ if (G.ar.nm > 1 && (!hmap(G.ar.data, (size_t) G.ar.nm * G.ar.slots * G.ar.stride, err) ||+ !hmap(G.ar.flags, (size_t) G.ar.nm * G.ar.slots * 32, err)))+ return false;+ // three members or more: the reduce-scatter's sums and its per-block flags (allreduce)+ if (G.ar.nm > 2 && (!hmap(G.ar.sums, (size_t) G.ar.slots * G.ar.stride, err) ||+ !hmap(G.ar.bflags, (size_t) 2 * G.ar.nm * G.ar.slots * kArBlocks * 8, err)))+ return false;+ if (G.ar.nm > 1 && &G == &groups[(size_t) devs[(size_t) head_dev].grp] &&+ (!hmap(G.lg_stage, (size_t) 2 * (G.ar.nm - 1) * kWin * ((g.vocab / 128 + G.ar.nm - 1) / G.ar.nm * 128), err) ||+ !hmap(G.lg_flag, (size_t) 2 * (G.ar.nm - 1) * 32, err) || !hmap(G.lg_ack, (size_t) 2 * 32, err)))+ return false;+ G.arb = G.ar; // (a chunk's: two slots of chunk rows)+ G.arb.slots = 2;+ G.arb.stride = (size_t) chunk * g.dim;+ if (chunk > 0 && G.arb.nm > 1 && (!hmap(G.arb.data, (size_t) G.arb.nm * G.arb.slots * G.arb.stride, err) ||+ !hmap(G.arb.flags, (size_t) G.arb.nm * G.arb.slots * 32, err)))+ return false;+ if (chunk > 0 && G.arb.nm > 2 &&+ (!hmap(G.arb.sums, (size_t) G.arb.slots * G.arb.stride, err) ||+ !hmap(G.arb.bflags, (size_t) 2 * G.arb.nm * G.arb.slots * kArBlocks * 8, err)))+ return false;+ }+ if (g.n_mtp > 0) {+ if (!halloc(h_taps, (size_t) tap_cap * g.d_targets.size() * g.dim, err)) return false;+ CK(cudaSetDevice(devs[(size_t) d_dev].id));+ CK(cudaEventCreateWithFlags(&ev_seed, cudaEventDisableTiming));+ CK(cudaEventRecord(ev_seed, devs[(size_t) d_dev].st));+ }+ // the dense weights, each device's layers by a thread of its own+ {+ std::vector<std::string> errs(devs.size());+ std::vector<std::thread> th;+ for (size_t di = 0; di < devs.size(); ++di)+ th.emplace_back([this, di, &errs] {+ std::string e;+ if (!load_dense((int) di, e)) errs[di] = e;+ });+ for (auto& t : th) t.join();+ for (auto& e : errs)+ if (!e.empty()) {+ err = e;+ return false;+ }+ }+ if (o.verbose)+ std::fprintf(stderr, "ds4: dense weights loaded in %.1f s\n", (now_ms() - t0) / 1000.0);+ // the markov steps over a head group of two or more (DS4_MK_SPLIT=0: the lead's whole rows, as before): E+ // host-mapped, the steps' slots+ // (only where every member holds its rows of the markov head: a type load_dense does not take keeps the lead's)+ bool mk_rows = g.n_mtp > 0 && head_nm() > 1;+ for (int m : groups[(size_t) devs[(size_t) head_dev].grp].mem) mk_rows = mk_rows && devs[(size_t) m].mk_share != nullptr;+ if (mk_rows && !(std::getenv("DS4_MK_SPLIT") != nullptr && std::atoi(std::getenv("DS4_MK_SPLIT")) == 0)) {+ const Tensor* te = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.embed.weight");+ if (te == nullptr || te->dtype != "BF16" || te->bytes != (uint64_t) g.vocab * g.d_rank * 2) {+ err = "the markov embed: not a BF16 [vocab][rank] table";+ return false;+ }+ if (!hmap(h_mk_embed, (size_t) g.vocab * g.d_rank, err) || !pk.read(*te, h_mk_embed) ||+ !hmap(mk_slots, (size_t) kWin * head_nm() * 4, err)) {+ if (err.empty()) err = "the markov embed: read failed";+ return false;+ }+ devs[(size_t) head_dev].mk_ints = d_ints; // (the lead's drafted tokens: where the confidence head reads them)+ mk_split = true;+ }++ // the embedding table, pinned on the host+ {+ const Tensor* t = pk.get("embed.weight");+ const size_t nv = (size_t) g.vocab * g.dim;+ const bool q8 = t != nullptr && t->dtype == "Q8_0" && t->bytes == nv / 32 * gg::kQ8Bytes;+ // other block types (a GGUF whose dense tensors are Q4_K / Q5_K / Q6_K: the IST-DAS sets' Q5_K embed.weight):+ // each row through ggml's to_float, to the same BF16 the host table holds+ const int ety = t != nullptr ? gg_type_of(t->dtype) : -1;+ const size_t erb = ety > 0 ? gg::type_row_bytes(ety, g.dim) : 0;+ const bool qrow = t != nullptr && !(t->dtype == "BF16") && erb > 0 && t->bytes == (size_t) g.vocab * erb;+ if (t == nullptr || (!q8 && !qrow && (t->dtype != "BF16" || t->bytes != nv * 2))) {+ err = "embed.weight: not a BF16 (or Q8_0 / a ggml block type) [vocab][dim] table";+ return false;+ }+ if (!halloc(embed_pinned, nv, err)) return false;+ if (qrow) { // (ggml's decode of each row, rounded to BF16 as the Q8_0 path rounds)+ std::vector<uint8_t> row(erb);+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {+ if (!pk.read_at(t->file, t->off + r * erb, erb, row.data())) {+ err = "embed.weight: read failed";+ return false;+ }+ if (!gg::gg_row_to_bf16(ety, row.data(), g.dim, embed_pinned + r * g.dim)) {+ err = "embed.weight: ggml has no decode for " + t->dtype;+ return false;+ }+ }+ } else if (q8) { // (a GGUF's: each value rounded to BF16, a row at a time)+ std::vector<uint8_t> row((size_t) g.dim / 32 * gg::kQ8Bytes);+ for (size_t r = 0; r < (size_t) g.vocab; ++r) {+ if (!pk.read_at(t->file, t->off + r * row.size(), row.size(), row.data())) {+ err = "embed.weight: read failed";+ return false;+ }+ const std::vector<float> v = q8_values(row.data(), (size_t) g.dim);+ for (size_t c = 0; c < (size_t) g.dim; ++c) {+ uint32_t u;+ std::memcpy(&u, &v[c], 4);+ u += 0x7fffu + ((u >> 16) & 1u); // round to nearest even+ embed_pinned[r * g.dim + c] = (uint16_t) (u >> 16);+ }+ }+ } else if (!pk.read(*t, embed_pinned)) {+ err = "embed.weight: read failed";+ return false;+ }++ if (std::getenv("DS4_DBG_EMB")) std::fprintf(stderr,++ "DBG emb pinned: %04x %04x %04x %04x (row1: %04x %04x)\n",++ (unsigned) embed_pinned[0], (unsigned) embed_pinned[1],++ (unsigned) embed_pinned[2], (unsigned) embed_pinned[3],++ (unsigned) embed_pinned[(size_t) g.dim], (unsigned) embed_pinned[(size_t) g.dim + 1]);+++ }+ // the host side of the CPU lane and the Engram rows+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads;+ if (!halloc(h_x, (size_t) kWin * g.dim, err) || !halloc(h_w, (size_t) kWin * kPicks, err) ||- !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) ||++ !halloc(h_part, (size_t) kWin * g.dim, err) || !halloc(h_idx, (size_t) kWin * kPicks, err) || !halloc(h_idx_a, (size_t) 2 * kPicks, err) ||+ !halloc(h_emb, (size_t) std::max(kWin, chunk) * g.dim, err) ||+ !halloc(h_eg, (size_t) std::max(1, pk.eg_layers) * eg_cap * cols * (pk.eg_hd + pk.eg_hd / 32) + 64, err))+ return false;+ if (pk.eg_layers > 0) {+ // the Engram rows' reader: threads enough to keep an SSD's queue busy (reads from the page cache need few);+ // the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)+ const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;+#if defined(_WIN32)+ // (Windows: each thread's ReadFile is synchronous, so the threads are the queue depth - Project Maya: a 28K-token+ // prompt in one chunk, 10 threads 17.1 s, 64 13.4 s on a 20-core PC. Here (6 cores, shipped profile) 64 readers+ // gave +4.5% on a 4K prompt but no decode gain in every pair: DS4_ENGRAM_THREADS=64 turns them on)+#endif+ const unsigned hw = std::max(1u, std::thread::hardware_concurrency());+ int nth = (int) std::min(16u, std::max(4u, hw / 2));+ if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));+ const uint64_t avail = ram_avail();+ uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));+ if (const char* e = std::getenv("DS4_ENGRAM_CACHE_MB")) mb = std::strtoull(e, nullptr, 10);+ eg_cache.init((size_t) ((mb << 20) / row), row);+ std::vector<std::string> paths;+ for (const std::string& fn : pk.files) paths.push_back(pk.model_dir + "/" + fn);+ // Windows: the Engram tables' files that hold nothing else are unmapped and read unbuffered - an SSD's+ // queue kept by the reader threads instead of the page cache's (whose reads of one file from many threads+ // went ~255K rows a second here); unbuffered reads of a mapped file take a slow path, so only unmapped+ // ones (off by default: -8% on a 4K prompt here; DS4_EG_DIRECT=1 turns it on)+ std::vector<char> eg_direct(pk.files.size(), 0);+#if defined(_WIN32)+ if (pk.gguf && std::getenv("DS4_EG_DIRECT") != nullptr && std::atoi(std::getenv("DS4_EG_DIRECT")) != 0) {+ std::vector<char> eg_file(pk.files.size(), 0), other(pk.files.size(), 0);+ for (const auto& kt : pk.t)+ if (kt.second.file >= 0 && (size_t) kt.second.file < pk.files.size())+ (kt.first.find(".engram.embed.") != std::string::npos ? eg_file : other)[(size_t) kt.second.file] = 1;+ for (size_t i = 0; i < pk.files.size(); ++i)+ if (eg_file[i] && !other[i]) {+ if (i < pk.maps.size() && pk.maps[i].first != nullptr) {+ os::unmap(pk.maps[i].first, pk.maps[i].second);+ pk.maps[i] = {nullptr, 0};+ }+ eg_direct[i] = 1;+ if (o.verbose) std::fprintf(stderr, "ds4: %s unmapped, its Engram rows read unbuffered\n",+ pk.files[i].c_str());+ }+ }+#endif+ eg_io.start(paths, nth, eg_direct);+ for (int li = 0; li < pk.eg_layers; ++li) {+ eg_win.emplace_back(new EgBatch());+ const std::string id = std::to_string(g.eg_layers[(size_t) li]);+ const Tensor* tw = pk.get("layers." + id + ".engram.embed.weight");+ // The row layout the reader and the dequant take. A type not here (or no tensor at all) would be read+ // as some other format's bytes - the old code then read the scale tensor of a pack that has none and+ // crashed in eg_issue - so refuse the pack once, with the type's name, before any token is read.+ int fmt;+ if (tw == nullptr) {+ std::fprintf(stderr, "ds4: Engram layer %s: engram.embed.weight is missing from the pack\n", id.c_str());+ std::exit(2);+ } else if (tw->dtype == "MXFP8") {+ fmt = 0;+ } else if (tw->dtype == "Q6_K") {+ fmt = 1;+ } else if (tw->dtype == "IQ4_NL") {+ fmt = 2;+ } else if (pk.get("layers." + id + ".engram.embed.scale") != nullptr) {+ fmt = 3; // (an old pack: FP8 codes in one tensor, their E8M0 scales in another)+ } else {+ std::fprintf(stderr,+ "ds4: Engram layer %s: engram.embed.weight type %s is not one the Engram reader handles "+ "(MXFP8, Q6_K, IQ4_NL)\n",+ id.c_str(), tw->dtype.c_str());+ std::exit(2);+ }+ if (fmt == 1 && pk.eg_hd % 256 != 0) {+ err = "Engram layer " + id + ": Q6_K rows of " + std::to_string(pk.eg_hd) ++ " values (not whole superblocks of 256)";+ return false;+ }+ if (fmt == 2 && pk.eg_hd % 32 != 0) {+ err = "Engram layer " + id + ": IQ4_NL rows of " + std::to_string(pk.eg_hd) ++ " values (not whole blocks of 32)";+ return false;+ }+ eg_fmt.push_back((uint8_t) fmt);+ }+ if (o.verbose)+ std::fprintf(stderr, "ds4: Engram rows read on %d threads, issued with the tokens, collected before "+ "their layer; row cache %llu MB (%zu rows)\n",+ nth, (unsigned long long) mb, eg_cache.capacity());+ }+ prof.assign((size_t) (g.nl + g.n_mtp) * g.n_exp, 0u);+ if (!o.profile.empty()) {+ std::FILE* f = std::fopen(o.profile.c_str(), "rb");+ if (f != nullptr) {+ std::vector<uint32_t> p(prof.size());+ const size_t n = std::fread(p.data(), 4, p.size(), f);+ if (n == p.size() || n == (size_t) g.nl * g.n_exp) std::copy(p.begin(), p.begin() + (long) n, prof.begin());+ std::fclose(f);+ }+ }++ // the chunked prompt path's buffers (before the experts: they take what is left)+ if (chunk > 0) {+ // recommend, never force: a chunk the cards cannot hold in buffers is halved (down to 256 rows) and then the+ // prompt runs in windows of 16, said in one line - the load only fails for a reason that is not this+ std::vector<size_t> na0, sc0;+ for (Dev& d : devs) {+ na0.push_back(d.allocs.size());+ sc0.push_back(d.scratch);+ }+ const size_t nh = host_allocs.size();+ std::string aerr;+ while (chunk > 0 && !alloc_chunks(aerr)) {+ for (size_t di = 0; di < devs.size(); ++di) {+ Dev& d = devs[di];+ cudaSetDevice(d.id);+ cudaDeviceSynchronize();+ if (di < chunks.size() && chunks[di]) {+ if (chunks[di]->blas) cublasDestroy(chunks[di]->blas);+ if (chunks[di]->cs) cudaStreamDestroy(chunks[di]->cs);+ }+ while (d.allocs.size() > na0[di]) {+ cudaFree(d.allocs.back());+ d.allocs.pop_back();+ }+ d.scratch = sc0[di];+ }+ while (host_allocs.size() > nh) {+ cudaFreeHost(host_allocs.back());+ host_allocs.pop_back();+ }+ chunks.clear();+ if (slab_auto_grew_) { // the automatic slab did not fit: the 0.2.6 slab first, the chunk only if that does not fit either+ slab_auto_grew_ = false;+ slab_force_base_ = true;+ std::fprintf(stderr, "ds4: the automatic attention slab did not fit (%s): trying the 256-row slab\n", aerr.c_str());+ aerr.clear();+ continue;+ }+ const int was = chunk;+ chunk = chunk > 512 ? chunk / 2 / 16 * 16 : 0;+ if (chunk > 0) {+ eg_cap = (size_t) chunk;+ std::fprintf(stderr, "ds4: the prompt path's buffers for chunks of %d rows did not fit (%s): trying %d\n",+ was, aerr.c_str(), chunk);+ } else {+ prompt_par = false;+ eg_cap = (size_t) kWin;+ tap_cap = kWin;+ std::fprintf(stderr, "ds4: the prompt path's buffers did not fit (%s): prompts run in windows of %d "+ "tokens (slower on long prompts; more free GPU memory or a lower --reserve-mib "+ "brings the chunks back)\n", aerr.c_str(), kWin);+ }+ aerr.clear();+ }+ if (chunk > 0 && aerr.empty()) err.clear();+ }+ // the CPU lane's experts read in place (decided above): how much of them the load reads ahead, from what VRAM+ // now leaves over+ if (file_experts) {+ double room = 0, all = 0; // the VRAM place_experts will find for experts; every expert's bytes+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ size_t fr = 0, tot = 0;+ CK(cudaMemGetInfo(&fr, &tot));+ room += std::max(0.0, (double) fr - (double) ((size_t) o.reserve_mib << 20)) * o.vram_experts;+ }+ for (const Layer& l : L) all += (double) l.blob * l.n_exp;+ const double need = std::max(0.0, all - room);+ const uint64_t avail = ram_avail();+ warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;+ if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;+ if (store_gb() > 0) warm_bytes = 0; // (the store is the RAM tier: nothing to warm in the OS cache)+ std::fprintf(stderr,+ "ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s "+ "(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",+ need / 1e9, o.mmap_experts > 0 ? " (--mmap-experts on)"+ : pk.gguf ? " (--mmap-experts off copies them into RAM)" : ", not copied into RAM",+ avail == ~uint64_t{0} ? 0.0 : avail / 1e9, warm_bytes / 1e9, chunk);+ }+ // the CPU lane: one pool of workers per NUMA node, each expert split in NH column parts (part X on node X); in+ // place, one part (the files' layout) and one pool over every node's cores (the pages lie where the cache put them)+ cores = node_cores();+ NH = file_experts ? 1 : (int) std::min<size_t>(2, cores.size());+ if (NH > 1 && ((g.ff / NH) % 128 != 0 || (g.dim / NH) % 128 != 0)) NH = 1;+ if (!pk.gguf) make_pools(); // (a GGUF's after its experts: copy_experts decides NH)+ {+ const size_t E = (size_t) g.dim, FF = (size_t) g.ff, GQ = E / 256, DQ = FF / 128, S = kSlots;+ c_xh.resize((size_t) NH * S * 2 * E);+ c_xsb.resize((size_t) NH * S * 2 * (E / 128));+ c_acc.resize((size_t) NH * S * 2 * GQ * (FF / NH));+ c_xhd.resize(S * FF);+ c_xsd.resize(S * (FF / 128));+ c_accd.resize((size_t) NH * S * DQ * (E / NH));+ c_yd.resize(S * E);+ }+ // the experts: VRAM slots in what each device has left, the rest on the host+ if (!place_experts(err)) return false;+ if (!load_experts(err)) return false;+ if (pk.gguf) make_pools();+ if (chunk > 0) {+ // the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slots+ const double tr = now_ms();+ for (auto& m : host_maps)+ if (cudaHostRegister(m.first, m.second, cudaHostRegisterPortable) != cudaSuccess) {+ err = "pinning the CPU lane's experts (" + std::to_string(m.second >> 20) + " MiB) failed";+ return false;+ }+ if (o.verbose) std::fprintf(stderr, "ds4: CPU lane's experts pinned in %.1f s\n", (now_ms() - tr) / 1000.0);+ if (file_experts) { // in place: each device's staging for its ring, and its readers (DS4_READERS, 8)+ size_t mr = 0;+ for (const Layer& l : L)+ if (!l.mtp) {+ for (uint64_t r : l.region_bytes) mr = std::max<size_t>(mr, r);+ if (l.lay.fmt == gg::kFmtMXFP4) { // (a GGUF's: read as its blob)+ mr = std::max(mr, l.blob);+ for (int r = 0; r < 3; ++r) flow_part[r] = std::max(flow_part[r], mx_part_bytes(l, r));+ }+ }+ const char* nr = std::getenv("DS4_READERS");+ const int n_readers = std::max(1, nr != nullptr ? std::atoi(nr) : 8);+ const bool flow_on = std::getenv("DS4_FLOW") == nullptr || std::atoi(std::getenv("DS4_FLOW")) != 0;+ // (a tensor-parallel group's members on the flow only with DS4_FLOW_TP=1 for now: each reads the+ // whole down of every expert it streams, 25 MB an expert for a pair, 18.8 alone - 3x V100, the+ // 17-token prompt from the page cache: 2.5-2.7 s on the flow, 1.3-1.6 s in the group's windows)+ const bool flow_tp = std::getenv("DS4_FLOW_TP") != nullptr && std::atoi(std::getenv("DS4_FLOW_TP")) != 0;+ for (size_t di = 0; di < chunks.size(); ++di) {+ auto& cp = chunks[di];+ if (cp == nullptr || cp->ring == nullptr) continue;+ cp->stage_stride = (mr + 4095) & ~(size_t) 4095;+ // the flow: every trunk layer of the device a GGUF's (alone or a tensor-parallel group's member)+ bool fl = flow_on;+ int n_exp_max = 0;+ for (const Layer& l : L)+ if (!l.mtp)+ for (int m = 0; m < l.nm; ++m)+ if (member(l, m).dev == (int) di) {+ // (a group's members: each its share of the down's blocks in every row)+ fl = fl && l.lay.fmt == gg::kFmtMXFP4 &&+ (l.nm == 1 || (flow_tp && (g.ff / l.nm) %+ std::max(1, gg::type_block(gg::expert_type(l.lay, 2))) == 0));+ n_exp_max = std::max(n_exp_max, l.n_exp);+ }+ if (fl) // (each part read unbuffered from its offset rounded down: a page more each)+ cp->stage_stride = std::max(cp->stage_stride, flow_stride());+ cp->S = cp->ring_n;+ if (!halloc(cp->stage, (size_t) cp->S * cp->stage_stride, err)) return false;+ cp->readers = std::make_unique<Pool>(n_readers - 1, std::vector<int>{}, 50);+ if (!fl) continue;+ cudaSetDevice(devs[di].id);+ cp->flow = true;++#if defined(STRATA_METAL)++ cp->direct = std::getenv("DS4_FLOW_DIRECT") == nullptr || std::atoi(std::getenv("DS4_FLOW_DIRECT")) != 0;++#endif+ cp->dev = (int) di;+ cp->io = std::make_unique<IoQueue>(n_readers);+ cp->stage_item.assign((size_t) cp->S, -1);+ cp->stage_ev.resize((size_t) cp->S);+ cp->ring_fill.resize((size_t) cp->ring_n);+ cp->ring_free.resize((size_t) cp->ring_n);+ for (auto* v : {&cp->stage_ev, &cp->ring_fill, &cp->ring_free})+ for (cudaEvent_t& e : *v) CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));+ // (a region is written again only bc_n batches later - more than a layer's items, so after that+ // layer's bounds wait, which every copy from it precedes)+ cp->bc_n = n_exp_max + 2;+ cp->bc_w = cp->ring_n + 1;+ if (!halloc(cp->bc_pin, (size_t) cp->bc_n * cp->bc_w, err) ||+ !dalloc(devs[di], cp->bc_dev, (size_t) cp->bc_n * cp->bc_w, err))+ return false;+ }+ // every device with layers on the flow: a prompt reads only the experts it picks, once, so a short one+ // takes the chunked path too (one pass, where windows of 16 read the CPU lane's picks again for each+ // window, a page fault at a time) - DS4_CHUNK_MIN still overrides+ bool all_flow = true;+ for (const auto& cp : chunks) all_flow = all_flow && (cp == nullptr || cp->ring == nullptr || cp->flow);+ if (all_flow && std::getenv("DS4_CHUNK_MIN") == nullptr) chunk_min = kFlowChunkMin;+ }+ for (Layer& l : L) {+ if (l.mtp) continue;+ for (int m = 0; m < l.nm; ++m) { // (each member its own: the flow's members decide them each)+ Layer& x = member(l, m);+ if (!halloc(x.vs_pin, (size_t) l.n_exp, err)) return false;+ if (!dalloc(devs[(size_t) x.dev], x.vs_dev, (size_t) l.n_exp, err)) return false;+ }+ }+ }+ {+ void* m = nullptr;+ if (cudaHostAlloc(&m, sizeof(Req) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {+ err = "host-mapped memory for the decode path's requests";+ return false;+ }+ host_allocs.push_back(m);+ reqs = (Req*) m;+ if (cudaHostAlloc(&m, sizeof(Resp) * kRing, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {+ err = "host-mapped memory for the decode path's answers";+ return false;+ }+ host_allocs.push_back(m);+ resps = (Resp*) m;+ std::memset((void*) reqs, 0, sizeof(Req) * kRing);+ std::memset((void*) resps, 0, sizeof(Resp) * kRing);+ for (auto& x : layer_of) x.store(-1);+ ahead_fl.assign(L.size() * 1024, 0);+ ahead_rank.assign(L.size() * 1024, 0);+ ahead_for.assign(L.size(), 0);+ const int core = !cores.empty() && !cores[0].empty() ? cores[0][0] : -1;+ service = std::thread([this, core] {+ if (core >= 0) pin_self(core);+ serve_lane();+ });+ if (ahead_mode > 0) pf_th = std::thread([this] { prefetch_loop(); });+ }+ if (o.expert_cache) {+ size_t maxb = 0;+ for (const Layer& l : L) maxb = std::max(maxb, l.blob);+ const int nt = std::max(1, std::min(c_inflight, std::getenv("DS4_CACHE_THREADS") ? std::atoi(std::getenv("DS4_CACHE_THREADS")) : 1));+ sw_bufs.resize((size_t) nt);+ for (SwapBufs& B : sw_bufs) {+ if (!halloc(B.a, maxb, err) || !halloc(B.b, maxb, err) || !halloc(B.h, maxb, err)) return false;+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ cudaStream_t cs = nullptr;+ CK(cudaStreamCreateWithFlags(&cs, cudaStreamNonBlocking));+ B.st.push_back(cs);+ }+ }+ for (SwapBufs& B : sw_bufs) cache_ths.emplace_back([this, &B] { cache_loop(B); });+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ CK(cudaStreamSynchronize(d.st));+ }+ if (par || chunk > 0) // (the windows' threads; the prompt path's, whatever mode the windows take)+ for (size_t di = 0; di < devs.size(); ++di) {+ workers.push_back(std::make_unique<Worker>());+ workers.back()->start();+ }+ calibrate_paths();+ init_store(); // (last: the store's arena is registered with the driver - page-locked memory the other pinned allocations must come before)+ preload_store();+ if (o.verbose) std::fprintf(stderr, "ds4: loaded in %.1f s\n%s", (now_ms() - t0) / 1000.0, describe().c_str());+ if (chunk > 0) (void) pick_flow_min(); // (the startup line)+ reset();+ start_warm();+ return true;+ }+++ // the caches' row bytes, packed as DeepSeek keeps them (ds4_kernels.cuh): the window ring's (FP8), the compressed+ // KV's (FP4 + E4M3 scales), the index keys' (FP4 + power-of-two scales)+ size_t win_rb() const { return kv_row_bytes(kKvFp8, g.hd); }+ size_t ckv_rb() const { return kv_row_bytes(kKvFp4, g.hd); }+ size_t ik_rb() const { return kv_row_bytes(kKvFp4p, g.ix_hd); }++ // The chunked prompt's first GEMM of each kind pays the BLAS library's start (its handle's heuristics, the code objects of its+ // kernels loaded on first use): ~1 s of hipModuleLoad plus ~2.5 s of library time on the R9700 (rocprofv3), 1-2 s on CUDA, once a+ // process - 3.6 s on the first prompt that went through the flow, measured as 4.9 s for a 25-token pass against 1.0 s for the+ // next one. Paid here instead, at load, on a thread beside the store's read: the chunk GEMMs on tiny inputs into the chunk's own+ // scratch buffers (the mHC mix GEMM in F32, the FP16 GEMM the projections and the indexer weights use) - no engine state is+ // touched (heat, profile, stats, positions, KV), so the answers are what they were. DS4_WARM=0 turns it off.+ void warm_flow() {+ const int T = 16, D = g.dim, hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;+ const float one = 1.0f, zero = 0.0f;+ for (size_t di = 0; di < chunks.size(); ++di) {+ if (!chunks[di] || chunks[di]->blas == nullptr) continue;+ Chunk& c = *chunks[di];+ if (c.C < T) continue;+ cudaSetDevice(devs[di].id);+ (void) cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, c.h, hcd, c.h, hcd, &zero, c.mix, 32);+ const int ns[2] = {256, 32};+ for (int n : ns)+ (void) cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, T, D, &one, c.w16, CUDA_R_16F, D, c.x16, CUDA_R_16F, D, &zero,+ c.y, CUDA_R_32F, n, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);+ (void) cudaStreamSynchronize(devs[di].st);+ (void) cudaGetLastError();+ }+ }+ // The warm-up runs once the model is loaded, on a thread of its own: HIP serialises the BLAS start with the store's page-locked+ // arena (measured: beside the store's read it finished only when the read did, 21 s, and load still grew by its 3.5 s), so+ // it cannot hide inside the load; after it, a person typing gives it the seconds it needs. A chunked pass that starts first+ // waits for it (warm_mu) - never later than that pass would have been without it - and a warm-up that finds the flow already+ // used does nothing. DS4_WARM=0 turns it off.+ std::mutex warm_mu;+ std::atomic<bool> flow_used{false};+ std::thread warm_bg;+ void start_warm() {+ if (chunk <= 0 || (std::getenv("DS4_WARM") && std::atoi(std::getenv("DS4_WARM")) == 0)) return;+ warm_bg = std::thread([this] {+ std::lock_guard<std::mutex> lk(warm_mu);+ if (flow_used.load() || quit.load()) return;+ const double tw = now_ms();+ warm_flow();+ if (o.verbose) std::fprintf(stderr, "ds4: prompt path warmed up in %.0f ms\n", now_ms() - tw);+ });+ }+ // ---- the chunked prompt path+ size_t x_cap() const { return (size_t) std::max<int>(kWin, chunk + 2); } // a window's / chunk's new source rows+ size_t chunk_hand_bytes() const {+ return (size_t) chunk * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);+ }++ // the staging ring's size (MB) for the experts a chunk streams from the CPU lane: 400 where a good part of them are+ // streamed; 16 where the cards hold most of the experts (they are within 30% of fitting) (a 4-card box) - the ring's VRAM is then worth more+ // as expert slots, and the decode after a prompt keeps the windowed path's speed (R9700 x4: 29.6 -> 31.7 tok/s)+ // The share of the routed experts the GPUs can hold, from what is known at layout (their free memory less the reserve, the+ // prompt path's buffers and the dense weights; the trunk's experts only when no drafter is loaded): 1 = all of them.+ double gpu_expert_share(double* exp_out = nullptr, double* room_out = nullptr, double* other_out = nullptr) const {+ double exp_b = 0, other_b = 0, room = 0;+ for (const auto& kt : pk.t) {+ const std::string& n = kt.first;+ if (n.find(".engram.embed.") != std::string::npos || n.rfind("vision.", 0) == 0 || n == "embed.weight" ||+ (!o.dspark && n.rfind("mtp.", 0) == 0))+ continue;+ (n.find(".ffn.experts.") != std::string::npos ? exp_b : other_b) += (double) kt.second.bytes;+ }+ for (const Dev& d : devs)+ room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) - 1.5 * (1 << 30)) * o.vram_experts;+ if (exp_out) *exp_out = exp_b;+ if (room_out) *room_out = room;+ if (other_out) *other_out = other_b;+ return exp_b > 0 ? std::min(1.0, std::max(0.0, room - other_b) / exp_b) : 1.0;+ }+ size_t ring_mb_auto() const {+ double exp_b = 0, other_b = 0, room = 0;+ const double share = gpu_expert_share(&exp_b, &room, &other_b);+ // One rule for the staging ring (ds4_experts_rule.hpp: staging_ring_mb): 16 MB where the cards hold the experts, a small+ // ring from each device's room where they hold most of them (4 x R9700), the larger streamed ring from the summed room on a+ // unified-memory GPU (Strix Halo: 400 -> 1200 MB) and where the cards hold under half. DS4_RING_MB overrides.+ bool unified = false;+ for (const Dev& d : devs) {+ cudaDeviceProp ip{};+ if (cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0) unified = true;+ }+ const size_t mb = (size_t) strata::ds4::staging_ring_mb(exp_b > 0 ? share : 0.0, room, (int) devs.size(), unified, rdna4(), defaults_026());+ if (o.verbose)+ std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,+ room / 1e9, other_b / 1e9, (int) mb);+ return mb;+ }+ bool slab_auto_grew_ = false, slab_force_base_ = false; // the automatic attention slab: grew this try / must not grow (it did not fit)+ bool alloc_chunks(std::string& err) {+ slab_auto_grew_ = false;+ chunks.resize(devs.size());+ const int D = g.dim, H = g.hc, HD = g.hd;+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;+ for (size_t di = 0; di < devs.size(); ++di) {+ Dev& d = devs[di];+ cudaSetDevice(d.id);+ chunks[di] = std::make_unique<Chunk>();+ Chunk& c = *chunks[di];+ const int nm = (int) groups[(size_t) d.grp].mem.size();+ const int NHL = g.heads / head_members(nm), HQ = NHL * HD, OG = g.o_groups / head_members(nm);+ int FF = 0, ff0_ = 0;+ split128(g.ff, nm, 0, ff0_, FF); // (the widest member's share: member 0 takes the odd blocks)+ const size_t C = (size_t) chunk;+ c.C = chunk;+ // the base slab: STRATA_DS4_A_MB megabytes of q. Every dense GEMM of a layer is issued once a slab, so the slab width divides+ // the prompt's dense weight traffic. A unified-memory GPU (Strix Halo, main 3133160ed) takes 32 MB: measured 8 -> 32 MB took+ // its 32K prompt 325.89 -> 357.78 tok/s, 16K 315.01 -> 328.04, 8K +1.6%, 4K flat, decode unmoved. A discrete card keeps 8 MB+ // (the 0.2.6 slab): its VRAM is the expert tier's, and on the RTX 5070 a bigger slab bought +2% prompt for -3% decode.+ // STRATA_DS4_A_MB overrides the base on any GPU (8 restores the old slab).+ bool unified = false;+ {+ cudaDeviceProp ip{};+ unified = cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0;+ }+ static const size_t a_mb_env = [] {+ const char* v = std::getenv("STRATA_DS4_A_MB");+ return (size_t) (v != nullptr ? std::max(1, std::atoi(v)) : 0);+ }();+ const size_t a_mb = strata::ds4::attn_slab_base_mb(unified, a_mb_env, defaults_026());+ c.A = std::max(64, (int) ((a_mb << 20) / (size_t) HQ) / 64 * 64);+ // DS4_ATTN_SLAB=N: the attention slab's rows. The dense projections (q_b, wo_a/b) are dequantized and GEMM'd+ // once a slab, so a slab of 256 rows dequantizes the same weights 32 times for an 8K prompt (measured: the+ // dense dequant 916 ms of the 8K prompt, rocBLAS 867 ms); a larger slab costs VRAM for c.q/c.o and the oa+ // buffers (A * HQ x 4 bytes each). 0.2.7: unset = automatic (attn_slab_rows: up to 2048 rows where the buffers+ // take VRAM no expert would have used), 0 = the 0.2.6 rule above, N = that many rows.+ {+ int env = -1;+ if (const char* v = std::getenv("DS4_ATTN_SLAB")) env = std::max(0, std::atoi(v));+ if ((slab_force_base_ || unified || a_mb_env != 0) && env < 0) env = 0; // (no automatic growth on a unified-memory GPU, or when STRATA_DS4_A_MB chose the base)+ // per row: c.q + c.o (float), c.o16 (half), c.oa (float), c.oa16 (half)+ const double bpr = (double) HQ * 10.0 + (double) OG * (double) g.o_lora * 6.0;+ double spare = 1e18; // the expert tier off (--vram-experts 0): the buffers cost no expert+ double share = 0.0; // the share of the routed experts the cards hold (0 with the tier off)+ if (o.vram_experts > 0.0) {+ double exp_b = 0, room = 0, other_b = 0;+ share = gpu_expert_share(&exp_b, &room, &other_b);+ spare = (room - other_b - exp_b) / (double) std::max<size_t>(1, devs.size());+ }+ const int base = c.A;+ // one rule (ds4_experts_rule.hpp: attn_slab_rows): the chunk's whole slab (<= 1024 rows) where the cards hold half the+ // experts or more (4 x R9700, +9% prompt), up to 2048 rows where the VRAM is spare, the base otherwise+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr, rdna4() ? share : 0.0, defaults_026());+ if (env < 0 && c.A > base) {+ slab_auto_grew_ = true;+ if (di == 0 && o.verbose) {+ if (rdna4() && share >= 0.5)+ std::fprintf(stderr, "ds4: attention slab %d rows (automatic: the cards hold %.0f%% of the experts; DS4_ATTN_SLAB=0 keeps %d)\n",+ c.A, share * 100.0, base);+ else+ std::fprintf(stderr, "ds4: attention slab %d rows (automatic: %.1f GB of VRAM to spare beyond the experts; DS4_ATTN_SLAB=0 keeps %d)\n",+ c.A, spare / 1e9, base);+ }+ }+ }+ // (the indexer's slab: its scores rows are the context wide - about 64 MB of them)+ c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));+ c.w16_elems = (size_t) 8 << 20;+ bool has_eg = false, has_first = false;+ size_t stride = 0;+ for (const Layer& l : L)+ for (int m = 0; m < l.nm; ++m)+ if (member(l, m).dev == (int) di && !l.mtp) {+ has_eg = has_eg || l.eg >= 0;+ has_first = has_first || l.il == 0;+ stride = std::max(stride, align256(unit(member(l, m))));+ }+ const size_t A = (size_t) c.A, AI = (size_t) c.AI, ss = (size_t) o.max_context + 64;+ bool ok = dalloc(d, c.h, C * H * D, err) && dalloc(d, c.x, C * D, err) && dalloc(d, c.acc, C * D, err) &&+ dalloc(d, c.x16, C * D, err) && dalloc(d, c.qr, C * g.q_lora, err) &&+ dalloc(d, c.qr16, C * g.q_lora, err) && dalloc(d, c.kvr, C * HD, err) &&+ dalloc(d, c.ckr, C * HD, err) && dalloc(d, c.cgr, C * HD, err) &&+ dalloc(d, c.latent, (C + 2) * HD, err) && dalloc(d, c.lat16, (C + 2) * HD, err) &&+ dalloc(d, c.ikr, (C + 2) * g.ix_hd, err) && dalloc(d, c.ext, ((size_t) g.win + C) * HD, err) &&+ dalloc(d, c.q, A * HQ, err) && dalloc(d, c.o, A * HQ, err) && dalloc(d, c.o16, A * HQ, err) &&+ dalloc(d, c.oa, A * OG * g.o_lora, err) && dalloc(d, c.oa16, A * OG * g.o_lora, err) &&+ dalloc(d, c.iq, AI * g.ix_heads * g.ix_hd, err) && dalloc(d, c.iw, C * 64, err) &&+ dalloc(d, c.scores, AI * ss, err) && dalloc(d, c.bm, AI * bm_stride(), err) &&+ dalloc(d, c.topk, C * topk_stride(), err) && dalloc(d, c.cand, C * cand_stride(), err) &&+ dalloc(d, c.mix, C * 32, err) && dalloc(d, c.a_pre, C * 8, err) && dalloc(d, c.a_post, C * 8, err) &&+ dalloc(d, c.a_comb, C * 64, err) && dalloc(d, c.f_pre, C * 8, err) &&+ dalloc(d, c.f_post, C * 8, err) && dalloc(d, c.f_comb, C * 64, err) && dalloc(d, c.pre, C * 8, err) &&+ dalloc(d, c.glog, C * 1024, err) && dalloc(d, c.ridx, C * kPicks, err) &&+ dalloc(d, c.rw, C * kPicks, err) && dalloc(d, c.bounds, 2048, err) &&+ dalloc(d, c.row_tok, C * kPicks, err) && dalloc(d, c.pos, C * kPicks, err) &&+ dalloc(d, c.sg, C * FF, err) && dalloc(d, c.su, C * FF, err) && dalloc(d, c.sh16, C * FF, err) &&- dalloc(d, c.h16, C * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&++ dalloc(d, c.h16, (C + 16) * FF, err) && dalloc(d, c.y, C * D, err) && dalloc(d, c.w16, c.w16_elems, err) &&+ halloc(c.bounds_pin, 2048, err);+ if (!ok) return false;+ if (has_first && !dalloc(d, c.emb, C * D, err)) return false;+ if (has_eg) {+ const size_t ES = 256, per = (size_t) cols * (ehd + ehd / 32);+ if (!dalloc(d, c.eg_x, ES * cols * ehd, err) || !dalloc(d, c.eg_x16, ES * cols * ehd, err) ||+ !dalloc(d, c.eg_kv, ES * (H + 1) * D, err) || !dalloc(d, c.eg_rows, C * per, err))+ return false;+ }+ if (stride > 0) { // DS4_RING_MB (400) of slots in kParts batches+ const size_t mb = std::getenv("DS4_RING_MB") ? (size_t) std::atoi(std::getenv("DS4_RING_MB")) : ring_mb_auto();+ c.ring_stride = stride;+ c.ring_n = std::max(Chunk::kParts, (int) ((mb << 20) / stride) / Chunk::kParts * Chunk::kParts);+ if (!dalloc(d, c.ring, (size_t) c.ring_n * stride, err)) return false;+ }+ CK(cudaStreamCreateWithFlags(&c.cs, cudaStreamNonBlocking));+ for (int i = 0; i < Chunk::kParts; ++i) {+ CK(cudaEventCreateWithFlags(&c.filled[i], cudaEventDisableTiming));+ CK(cudaEventCreateWithFlags(&c.freed[i], cudaEventDisableTiming));+ }+ if (cublasCreate(&c.blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(c.blas, d.st) != CUBLAS_STATUS_SUCCESS) {+ err = "cuBLAS on CUDA" + std::to_string(d.id);+ return false;+ }+ for (int sl = 0; sl < 2; ++sl) {+ if (d.mi == 0 && (!hmap(c.hand[sl], chunk_hand_bytes(), err) || !hmap(c.hand_flag[sl], 32, err))) return false;+ if (has_first && !halloc(c.emb_host[sl], C * D, err)) return false;+ if (has_eg && !halloc(c.eg_host[sl], C * pk.eg_layers * (size_t) cols * (ehd + ehd / 32), err)) return false;+ CK(cudaEventCreateWithFlags(&c.emb_ev[sl], cudaEventDisableTiming));+ CK(cudaEventCreateWithFlags(&c.eg_ev[sl], cudaEventDisableTiming));+ }+ if (d.mi == 0 && !hmap(c.hand_ack, (size_t) 2 * 8 * 32, err)) return false;+ }+ return true;+ }++ // y[r][c] = x16[r] . W[:, c] (+ beta y) for r < rows: W reconstructed to FP16 in whole 128-column blocks, then the product.+ // The product prefers the engine's own RDNA3 WMMA kernel (src/prefill/wmma_gemm.cu) over hipBLAS: on gfx1151 hipBLAS+ // measures 4.97 TFLOPS at this call convention while that kernel measures 20-40 (BM 128 x BN 256, grouped tile order), and+ // the chunked prompt's dense GEMM is 11.1 s of a 4K prompt at the library's rate. Same operands and the same values (the+ // k order differs, so it is rounding-level: the quality gate and the acceptance gate cover it); STRATA_DENSE_PFGEMM=0 keeps+ // hipBLAS for an A/B.+ bool gemm(Dev& d, Chunk& c, const x3::Mat& m, const uint16_t* x16, int ldx, int rows, float* y, int ldy, float beta,+ std::string& err) {+ if (rows <= 0) return true;+#if defined(STRATA_USE_HIP) // (wmma_gemm.cu is part of the AMD builds only: a CUDA build has no such kernel and keeps cuBLAS)+ static const bool pf = strata::ds4::dense_pfgemm_default(env_tri("STRATA_DENSE_PFGEMM"), defaults_026());+#endif+ const int cols = std::max(128, (int) std::min<size_t>((size_t) m.n, c.w16_elems / (size_t) m.k) / 128 * 128);+ for (int c0 = 0; c0 < m.n; c0 += cols) {+ const int nc = std::min(cols, m.n - c0);+ if (m.fmt != 0) gg::rows(nc == m.n ? m : gg::view(m, c0, nc), c.w16, false, d.st); // (a GGUF's)+ else x3::reconstruct_rows(nc == m.n ? m : x3::view(m, c0, nc), c.w16, false, d.st);+#if defined(STRATA_USE_HIP)+ if (pf && rows >= 64 && ((uintptr_t) x16 & 15u) == 0 &&+ strata_pf_gemm_f16_ld(x16, ldx, c.w16, m.k, y + c0, rows, nc, m.k, ldy, beta, d.st))+ continue;+#endif+ const float one = 1.0f;+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, nc, rows, m.k, &one, c.w16, CUDA_R_16F,+ m.k, x16, CUDA_R_16F, ldx, &beta, y + c0, CUDA_R_32F, ldy,+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP);+ if (st != CUBLAS_STATUS_SUCCESS) {+ err = "cublasGemmEx: status " + std::to_string((int) st);+ return false;+ }+ }+ return true;+ }+ // y[r][c] = alpha x16[r] . W[c] for F16 weights W [n][k] (the indexer's weights_proj)+ bool gemm_f16w(Chunk& c, const uint16_t* W, int n, int k, const uint16_t* x16, int ldx, int rows, float alpha,+ float* y, int ldy, std::string& err) {+ const float zero = 0.0f;+ const cublasStatus_t st = cublasGemmEx(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n, rows, k, &alpha, W, CUDA_R_16F, k,+ x16, CUDA_R_16F, ldx, &zero, y, CUDA_R_32F, ldy, CUBLAS_COMPUTE_32F,+ CUBLAS_GEMM_DEFAULT_TENSOR_OP);+ if (st != CUBLAS_STATUS_SUCCESS) {+ err = "cublasGemmEx (F16 weights): status " + std::to_string((int) st);+ return false;+ }+ return true;+ }+ // hc_mix over a chunk: the dots by an F32 GEMM, then the rms scale+ bool hc_mix_chunk(Dev& d, Chunk& c, const float* h, const float* fn, int T, float* mixes, std::string& err) {+ const int hcd = g.hc * g.dim, n_mix = (2 + g.hc) * g.hc;++#if defined(STRATA_METAL)++ if (!std::getenv("DS4_CHUNK_SGEMM")) { // the window path's fused kernel: no F32 GEMM, no second pass++ hc_mix(h, fn, hcd, n_mix, T, g.eps, mixes, d.st);++ return true;++ }++#endif+ const float one = 1.0f, zero = 0.0f;+ if (cublasSgemm(c.blas, CUBLAS_OP_T, CUBLAS_OP_N, n_mix, T, hcd, &one, fn, hcd, h, hcd, &zero, mixes, 32) !=+ CUBLAS_STATUS_SUCCESS) {+ err = "cublasSgemm (mHC mixes)";+ return false;+ }+ hc_mix_scale(h, hcd, n_mix, T, g.eps, mixes, d.st);+ return true;+ }++ // expert e of l (on the CPU lane: its NH column parts) -> member m's VRAM copy at dst, in the GPU layout (the full+ // blob, or a group member's part), by DMA from the pinned parts on the copy stream (in place: from `src`, its file+ // region read into pinned staging)+ bool stream_expert(const Layer& l, int m, int e, uint8_t* dst, cudaStream_t cs, std::string& err,+ const uint8_t* src = nullptr) {+ if (l.lay.fmt == gg::kFmtMXFP4) { // the blob, or member m's rows / blocks: from `src` (the expert read into+ // pinned staging as its blob) or from the file's mapping+ auto mat_r = [&](int r) { return src != nullptr ? src + l.lay.trellis[r] : mx_mat(l, e, r); };+ bool ok = true;+ if (l.nm == 1) {+ if (src != nullptr)+ ok = cudaMemcpyAsync(dst, src, l.blob, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ else+ for (int r = 0; r < 3; ++r)+ ok = ok && cudaMemcpyAsync(dst + l.lay.trellis[r], mat_r(r), mx_part_bytes(l, r),+ cudaMemcpyHostToDevice, cs) == cudaSuccess;+ } else {+ const int ffh = g.ff / l.nm;+ const size_t gr = gg::type_row_bytes(gg::expert_type(l.lay, 0), g.dim),+ dr = gg::type_row_bytes(gg::expert_type(l.lay, 2), g.ff),+ dh = gg::type_row_bytes(gg::expert_type(l.lay, 2), ffh);+ for (int r = 0; r < 2; ++r)+ ok = ok && cudaMemcpyAsync(dst + l.hlay.trellis[r], mat_r(r) + (size_t) m * ffh * gr,+ (size_t) ffh * gr, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ ok = ok && cudaMemcpy2DAsync(dst + l.hlay.trellis[2], dh, mat_r(2) + (size_t) m * dh, dr, dh,+ (size_t) g.dim, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ }+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";+ return ok;+ }+ const auto& parts = l.hh[(size_t) e];+ const int D = g.dim, FF = g.ff;+ auto cp2 = [&](uint8_t* d0, size_t dp, const uint8_t* s0, size_t sp, size_t w, size_t h) {+ return cudaMemcpy2DAsync(d0, dp, s0, sp, w, h, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ };+ auto cp = [&](uint8_t* d0, const uint8_t* s0, size_t n) {+ return cudaMemcpyAsync(d0, s0, n, cudaMemcpyHostToDevice, cs) == cudaSuccess;+ };+ bool ok = true;+ if (NH == 1) { // one part: the whole expert in exllamav3's layout, its pieces at hl's offsets+ const uint8_t* p = src != nullptr ? src : parts[0];+ if (l.nm == 1) {+ for (int r = 0; r < 3; ++r)+ ok = ok && cp(dst + l.lay.trellis[r], p + l.hl.trellis[r], l.piece_bytes[r][0]) &&+ cp(dst + l.lay.suh[r], p + l.hl.suh[r], (size_t) l.hl.k[r] * 2) &&+ cp(dst + l.lay.svh[r], p + l.hl.svh[r], (size_t) l.hl.n[r] * 2);+ } else { // member m's part (its FFN columns [f0, f0 + fn)), cut as blob_to_half cuts a blob+ const Layer& x = member(l, m);+ const int f0 = x.ff0, fn = x.ffn;+ for (int r = 0; r < 2; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) FF / 16, nth = (size_t) fn / 16;+ ok = ok && cp2(dst + x.hlay.trellis[r], nth * tile, p + l.hl.trellis[r] + (size_t) (f0 / 16) * tile,+ nt * tile, nth * tile, (size_t) D / 16) &&+ cp(dst + x.hlay.suh[r], p + l.hl.suh[r], (size_t) D * 2) &&+ cp(dst + x.hlay.svh[r], p + l.hl.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);+ }+ const size_t row = (size_t) (D / 16) * 32 * l.lay.K[2];+ ok = ok && cp(dst + x.hlay.trellis[2], p + l.hl.trellis[2] + (size_t) (f0 / 16) * row,+ (size_t) (fn / 16) * row) &&+ cp(dst + x.hlay.suh[2], p + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2) &&+ cp(dst + x.hlay.svh[2], p + l.hl.svh[2], (size_t) D * 2);+ }+ } else if (l.nm == 1) { // the full blob: each matrix's columns from the two parts side by side+ for (int r = 0; r < 3; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int k = l.hl.k[r], nh = l.hl.n[r]; // part: k x nh+ for (int X = 0; X < 2; ++X) {+ ok = ok && cp2(dst + l.lay.trellis[r] + (size_t) X * (nh / 16) * tile, (size_t) (2 * nh / 16) * tile,+ parts[(size_t) X] + l.hl.trellis[r], (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile,+ (size_t) (k / 16));+ ok = ok && cp(dst + l.lay.svh[r] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[r], (size_t) nh * 2);+ }+ ok = ok && cp(dst + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);+ }+ } else { // member m (FFN columns [f0, f0 + fn)): gate / up from the parts holding them; down rows [f0, ..) of+ // both parts' columns+ const Layer& x = member(l, m);+ const int f0 = x.ff0, fn = x.ffn;+ for (int r = 0; r < 2; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int hn = l.hl.n[r]; // part X: columns [X hn, (X + 1) hn)+ for (int X = 0; X < 2; ++X) {+ const int a = std::max(f0, X * hn), b = std::min(f0 + fn, (X + 1) * hn);+ if (a >= b) continue;+ ok = ok && cp2(dst + x.hlay.trellis[r] + (size_t) ((a - f0) / 16) * tile, (size_t) (fn / 16) * tile,+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) ((a - X * hn) / 16) * tile,+ (size_t) (hn / 16) * tile, (size_t) ((b - a) / 16) * tile, (size_t) (D / 16)) &&+ cp(dst + x.hlay.svh[r] + (size_t) (a - f0) * 2, parts[(size_t) X] + l.hl.svh[r] + (size_t) (a - X * hn) * 2,+ (size_t) (b - a) * 2);+ }+ ok = ok && cp(dst + x.hlay.suh[r], parts[0] + l.hl.suh[r], (size_t) D * 2);+ }+ const size_t tile = (size_t) 32 * l.lay.K[2];+ const int nh = D / 2;+ for (int X = 0; X < 2; ++X) {+ ok = ok && cp2(dst + x.hlay.trellis[2] + (size_t) X * (nh / 16) * tile, (size_t) (D / 16) * tile,+ parts[(size_t) X] + l.hl.trellis[2] + (size_t) (f0 / 16) * (nh / 16) * tile,+ (size_t) (nh / 16) * tile, (size_t) (nh / 16) * tile, (size_t) (fn / 16));+ ok = ok && cp(dst + x.hlay.svh[2] + (size_t) X * nh * 2, parts[(size_t) X] + l.hl.svh[2], (size_t) nh * 2);+ }+ ok = ok && cp(dst + x.hlay.suh[2], parts[0] + l.hl.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);+ }+ if (!ok) err = "streaming layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";+ return ok;+ }++ // a device's streamed experts: batches of ring_n / kParts of each of its layers' CPU-lane experts, chunk by chunk+ // in layer order; the copies run at most kParts batches ahead of the compute+ struct Batch {+ int k = 0, il = 0, j0 = 0, n = 0; // chunk, layer, its host_list range+ };+ struct Streamer {+ std::vector<Batch> b;+ size_t cp = 0, kp = 0; // the next batch to copy, to compute+ std::atomic<size_t> done{0}; // cp, for read_ahead's thread+ };+ // in place (file_experts): the batches' file regions read ahead of their copies - POSIX_FADV_WILLNEED, the kernel+ // reading them in the background in large requests, where the copies alone fault the pages in a few at a time -+ // at most DS4_READAHEAD_MB (2048) beyond the batch being copied; each region by its layer's first member alone+ void read_ahead(const Streamer& S, int di, const std::atomic<bool>& stop) const {+ static const double cap = (std::getenv("DS4_READAHEAD_MB") ? std::atof(std::getenv("DS4_READAHEAD_MB")) : 2048.0) *+ 1048576.0;+ std::deque<std::pair<size_t, double>> issued; // (batch, bytes) not yet copied+ double ahead = 0;+ for (size_t q = 0; q < S.b.size() && !stop.load(std::memory_order_relaxed);) {+ const size_t done = S.done.load(std::memory_order_relaxed);+ while (!issued.empty() && issued.front().first < done) {+ ahead -= issued.front().second;+ issued.pop_front();+ }+ if (ahead > cap) {+ std::this_thread::sleep_for(std::chrono::microseconds(500));+ continue;+ }+ const Batch& bt = S.b[q];+ const Layer& l0 = L[(size_t) bt.il];+ double nb = 0;+ if (l0.dev == di)+ for (int i = 0; i < bt.n; ++i) {+ const int e = l0.host_list[(size_t) (bt.j0 + i)];+ if (l0.lay.fmt == gg::kFmtMXFP4) {+ if (copy_mode && l0.ram[(size_t) e] >= 0) continue; // (in RAM: nothing to read)+ for (int r = 0; r < 3; ++r)+ os::advise(pk.fds[(size_t) l0.mx_file[r]], (l0.mx_off[r] + (uint64_t) e * l0.mx_stride[r]), mx_part_bytes(l0, r), true);+ nb += (double) l0.blob;+ continue;+ }+ os::advise(pk.fds[(size_t) l0.region_file[(size_t) e]], l0.region_off[(size_t) e], l0.region_bytes[(size_t) e], true);+ nb += (double) l0.region_bytes[(size_t) e];+ }+ issued.push_back({q++, nb});+ ahead += nb;+ }+ }+ bool copy_ahead(Dev& d, Chunk& c, Streamer& S, int di, std::string& err) {+ const int P = Chunk::kParts;+ while (S.cp < S.b.size() && S.cp < S.kp + P) {+ const Batch& bt = S.b[S.cp];+ const int half = (int) (S.cp % P);+ const Layer& l0 = L[(size_t) bt.il];+ int mi = 0;+ for (int m = 0; m < l0.nm; ++m)+ if (member(l0, m).dev == di) mi = m;+ if (S.cp >= (size_t) P) CK(cudaStreamWaitEvent(c.cs, c.freed[half], 0));+ uint8_t* sb = nullptr; // in place: the batch's regions in the part's staging slots+ if (c.stage != nullptr) {+ const double t_evt = now_ms();+ if (S.cp >= (size_t) P) CK(cudaEventSynchronize(c.filled[half])); // (its last batch's copies out)+ c.dbg_evt_ms += now_ms() - t_evt;+ sb = c.stage + (size_t) half * (c.ring_n / P) * c.stage_stride;+ std::atomic<bool> bad{false};+ const double t_st = now_ms();+ c.readers->run(bt.n, [&](int i) {+ const int e = l0.host_list[(size_t) (bt.j0 + i)];+ if (l0.lay.fmt == gg::kFmtMXFP4 ? !read_mx(l0, e, sb + (size_t) i * c.stage_stride)+ : !pk.read_at(l0.region_file[(size_t) e], l0.region_off[(size_t) e],+ l0.region_bytes[(size_t) e],+ sb + (size_t) i * c.stage_stride))+ bad.store(true);+ });+ c.dbg_stage_ms += now_ms() - t_st;+ if (bad.load()) {+ err = "reading layer " + std::to_string(l0.il) + "'s experts from the model's files failed";+ return false;+ }+ }+ const double t_dma = now_ms();+ for (int i = 0; i < bt.n; ++i) {+ uint8_t* dst = c.ring + ((size_t) half * (c.ring_n / P) + i) * c.ring_stride;+ if (!stream_expert(l0, mi, l0.host_list[(size_t) (bt.j0 + i)], dst, c.cs, err,+ sb != nullptr ? sb + (size_t) i * c.stage_stride : nullptr))+ return false;+ c.stream_bytes += (double) unit(member(l0, mi));+ }+ c.dbg_dma_ms += now_ms() - t_dma;+ CK(cudaEventRecord(c.filled[half], c.cs));+ ++S.cp;+ S.done.store(S.cp, std::memory_order_relaxed);+ }+ return true;+ }++ // ---- the flow (see Chunk)+ // Windows (WDDM: HIP's runtime and CUDA's driver alike) submits queued work only when the host waits on the GPU:+ // the flow's copies and the compute it queued between its waits are submitted at once (win_submit); elsewhere+ // nothing. (CUDA too since #303: two RTX 3080s hung before the first layer, one GPU spinning on the other's flow)+ static void kick(cudaStream_t s) { win_submit(s); }+ size_t flow_region(int r) const { // a part's room in a staging slot (read from its offset rounded down)+ return ((flow_part[r] + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1)) + os::kDirectAlign;+ }+ size_t flow_stride() const { return flow_region(0) + flow_region(1) + flow_region(2); }+ static Chunk::Item& fit(Chunk& c, size_t i) { return c.items[i - c.base]; }+ // what member m of l's group reads of matrix r of an expert: gate / up its rows [m n_ff / nm, ..) - one run, from+ // this offset in the matrix, this long; the down the whole matrix (its part, the blocks [m n_ff / nm / 32, ..) of+ // every row, is DMA'd out of it row by row); alone (nm 1) every matrix whole+ size_t mx_read_off(const Layer& l, int m, int r) const {+ return r < 2 && l.nm > 1 ? (size_t) m * (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)+ : 0;+ }+ size_t mx_read_len(const Layer& l, int r) const {+ return r < 2 && l.nm > 1 ? (size_t) (g.ff / l.nm) * gg::type_row_bytes(gg::expert_type(l.lay, r), g.dim)+ : mx_part_bytes(l, r);+ }+ // the device's member index in L0's group (0 alone)+ int flow_mi(const Chunk& c, const Layer& L0) const {+ for (int m = 0; m < L0.nm; ++m)+ if (member(L0, m).dev == c.dev) return m;+ return 0;+ }+ void flow_push(Chunk& c, const Layer& l, int mi, int k, int e, int v) {+ c.items.emplace_back();+ Chunk::Item& it = c.items.back();+ it.il = l.il;+ it.mi = mi;+ it.k = k;+ it.e = e;+ it.v = v;+ bool in_ram = !l.ram.empty() && l.ram[(size_t) e] >= 0;+ const uint8_t* store_blob = nullptr; // (the RAM store holds it: copied into staging like a RAM-tier slot, no read)+ if (!in_ram && store && l.nm == 1) {+ const int ss = store->find(l.il, e);+ if (ss >= 0) {+ store_blob = store->blob(ss);+ in_ram = true;+ }+ }+ if (in_ram && ram_pinned && store_blob == nullptr) { // (DMA'd from where it lies)+ const uint8_t* p = copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0]+ : ram_slots[(size_t) l.ram[(size_t) e]];+ for (int r = 0; r < 3; ++r)+ it.src[r] = p + (copy_mode ? cp_lay[(size_t) l.il][(size_t) r] : l.lay.trellis[r]) + mx_read_off(l, mi, r);+ ++c.it_ram;+ return;+ }+ if (in_ram) // (copied into staging by io, then DMA'd; copy_mode: from its NH parts, flow_pump)+ it.ram = store_blob != nullptr ? store_blob+ : copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] : ram_slots[(size_t) l.ram[(size_t) e]];+ bool hit = false;+ if (!in_ram && c.lc != nullptr && l.nm == 1) { // layer-major: from the layer cache, or kept for the chunks after+ const size_t i = c.base + c.items.size() - 1, stride = (l.blob + 63) & ~(size_t) 63;+ if (c.lc_il != l.il) { // (a layer's first item: the slots the layer before used are its own again+ c.lc_il = l.il; // once that layer's items are all read - `after` below)+ c.lc_used = 0;+ c.lc_n = (int) std::min<size_t>(c.lc_of.size(), c.lc_bytes / stride);+ c.lc_first = i;+ std::fill(c.lc_of.begin(), c.lc_of.end(), -1);+ }+ const int j = c.lc_of[(size_t) e];+ if (j >= 0) { // (copied into staging by io once its reader is done, then DMA'd)+ it.ram = c.lc + (size_t) j * stride;+ it.after = c.lc_writer[(size_t) j];+ hit = true;+ } else if (k + 1 < (int) c.fl_T.size() && c.lc_used < c.lc_n) { // (a later chunk may pick it)+ const int jn = c.lc_used++;+ c.lc_of[(size_t) e] = jn;+ c.lc_writer[(size_t) jn] = (int64_t) i;+ it.keep = c.lc + (size_t) jn * stride;+ it.after = (int64_t) c.lc_first - 1;+ }+ }++ if (c.direct && l.nm == 1 && !(in_ram && copy_mode)) { // (the ring slot is the destination: filled by flow_pump when its turn comes)++ it.direct = true;++ it.stage = -1;++ it.left.store(3, std::memory_order_relaxed);++ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);++ return;++ }+ it.stage = (int) (c.n_file++ % (uint64_t) c.S);+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;+ for (int r = 0; r < 3; ++r) {+ const uint64_t off = l.mx_off[r] + (uint64_t) e * l.mx_stride[r] + mx_read_off(l, mi, r);+ it.src[r] = sb + (in_ram || hit ? 0 : off % os::kDirectAlign);+ sb += flow_region(r);+ }+ it.left.store(in_ram && copy_mode && store_blob == nullptr ? 3 * NH : 3, std::memory_order_relaxed);+ ++(in_ram ? c.it_ram : hit ? c.it_lc : c.it_file);+ }+ // the likely experts of (chunk k, layer il) for T tokens, first in its stream: the CPU lane's experts whose chance+ // to be picked at least once, 1 - (1 - share)^(picks), is at least DS4_SPEC_Q (0.9: the reads are what takes the+ // time, an unpicked one costs a whole read, and the routing is known a few ms after the layer before is done -+ // early reads pay where a chunk's long attention keeps the disk otherwise idle, and there nearly every expert is+ // that likely), likeliest first, at most what the staging and the ring hold (no more can be read before the layer+ // routes); the layer's virtual slots put them first, then its other CPU-lane experts in index order (the order the+ // rest of its picks are streamed in)+ // (a tensor-parallel group's members each decide their own stream and virtual slots - the same, from the same+ // profile and the same routing - and each reads only its part of every expert)+ void flow_spec(Chunk& c, int k, int il, int T) {+ static const double spec_q = std::getenv("DS4_SPEC_Q") ? std::atof(std::getenv("DS4_SPEC_Q")) : 0.9;+ Layer& l = L[(size_t) il];+ const int mi = flow_mi(c, l);+ int* vs = member(l, mi).vs_pin;+ c.spec_k = k;+ c.spec_il = il;+ c.spec_begin = c.base + c.items.size();+ const uint32_t* pr = prof.data() + (size_t) il * g.n_exp;+ double sum = 0;+ for (int e = 0; e < l.n_exp; ++e) sum += pr[e];+ std::vector<std::pair<double, int>> cand;+ const double picks = (double) l.k_exp * T;+ // (early reads only while they hit: once a run's likely ones went unpicked more often than DS4_SPEC_WASTE+ // (0.2) of the time, the rest of the run reads only what the routing picked - an unpicked one costs a whole+ // read, and how well the profile guesses depends on the PC's history and the prompt: V100, 64 tokens - 376+ // of 921 likely ones unpicked, 7 GB of 84 read for nothing; Radeon 8065S, 128 and 512 tokens - 44 of 945 and+ // 21 of 1568, the early reads 9-12% faster there)+ static const double waste = std::getenv("DS4_SPEC_WASTE") ? std::atof(std::getenv("DS4_SPEC_WASTE")) : 0.2;+ const bool early = c.it_spec < 2u * (uint64_t) c.ring_n || (double) c.it_spec_unused <= waste * (double) c.it_spec;+ for (int e : l.host_list) {+ if (!early) break;+ const double p = sum > 0 ? pr[e] / sum : 0.0;+ const double q = p >= 1.0 ? 1.0 : 1.0 - std::exp(picks * std::log1p(-p));+ if (q >= spec_q && spec_q <= 1.0) cand.push_back({q, e});+ }+ std::stable_sort(cand.begin(), cand.end(), [](const std::pair<double, int>& a, const std::pair<double, int>& b) {+ return a.first > b.first;+ });+ if (cand.size() > (size_t) (c.S + c.ring_n)) cand.resize((size_t) (c.S + c.ring_n));+ std::vector<char> is((size_t) l.n_exp, 0);+ int j = 0;+ for (const auto& x : cand) {+ vs[x.second] = l.n_slots + j++;+ is[(size_t) x.second] = 1;+ }+ for (int e : l.host_list)+ if (!is[(size_t) e]) vs[e] = l.n_slots + j++;+ for (const auto& x : cand) flow_push(c, l, mi, k, x.second, vs[x.second]);+ c.n_spec = (int) cand.size();+ c.it_spec += cand.size();+ }+ // (chunk k, layer il) routed (bounds b by virtual slot): the rest of its picks after its likely ones+ void flow_rest(Chunk& c, int k, int il, const std::vector<int>& b) {+ const Layer& l = L[(size_t) il];+ const int mi = flow_mi(c, l);+ const int* vs = member(l, mi).vs_pin;+ for (size_t i = c.spec_begin; i < c.spec_begin + (size_t) c.n_spec; ++i) {+ const int v = fit(c, i).v;+ if (b[(size_t) v + 1] == b[(size_t) v]) ++c.it_spec_unused;+ }+ for (int e : l.host_list) { // (index order: their virtual slots ascending)+ const int v = vs[e];+ if (v >= l.n_slots + c.n_spec && b[(size_t) v + 1] > b[(size_t) v]) flow_push(c, l, mi, k, e, v);+ }+ }+ // the next (chunk, layer) of the device after (k, il) in the run's order that streams experts (false: none) - a+ // long prompt's skip leaves out its early chunks' layers from Run::yoco on (kv_only there: no experts)+ bool flow_next(const Chunk& c, int k, int il, int& k2, int& il2) const {+ while (flow_next1(c, k, il, k2, il2)) {+ if (c.fl_yoco < 0 || k2 >= c.fl_cut_k || il2 < c.fl_yoco) return true;+ k = k2;+ il = il2;+ }+ return false;+ }+ // the rows chunk k runs layer il on (a long prompt's skip: the cut chunk's from cut_r on, from the yoco layer on)+ static int fl_rows(const Chunk& c, int k, int il) {+ const int T = c.fl_T[(size_t) k];+ return c.fl_yoco >= 0 && k == c.fl_cut_k && il >= c.fl_yoco ? T - c.fl_cut_r : T;+ }+ bool flow_next1(const Chunk& c, int k, int il, int& k2, int& il2) const {+ const auto it = std::find(c.fl_layers.begin(), c.fl_layers.end(), il);+ const size_t i = (size_t) (it - c.fl_layers.begin());+ if (c.lm) { // (layer-major: the layer's next chunk, then the next layer's first)+ if (k + 1 < (int) c.fl_T.size()) {+ k2 = k + 1;+ il2 = il;+ return true;+ }+ if (i + 1 >= c.fl_layers.size()) return false;+ k2 = 0;+ il2 = c.fl_layers[i + 1];+ return true;+ }+ if (i + 1 < c.fl_layers.size()) {+ k2 = k;+ il2 = c.fl_layers[i + 1];+ return true;+ }+ if (k + 1 < (int) c.fl_T.size() && !c.fl_layers.empty()) {+ k2 = k + 1;+ il2 = c.fl_layers[0];+ return true;+ }+ return false;+ }+ // the reads (in order, each into its staging slot once that slot's last DMA is done) and the DMAs (in order, each+ // into its ring slot once that slot's last use is queued: the copy stream waits for it) as far as they can go now+ bool flow_pump(Chunk& c, std::string& err) {+ const size_t end = c.base + c.items.size();+ IoQueue::Job jobs[6];+ while (c.n_read < end) {+ Chunk::Item& it = fit(c, c.n_read);+ if (it.after >= 0) { // layer-major: a slot's expert copied out of it once its reader is done; a slot+ // written again once the layer before's items are all read (none still copies out of it)+ bool wait = false;+ if (it.keep == nullptr)+ wait = (size_t) it.after >= c.base && fit(c, (size_t) it.after).left.load(std::memory_order_acquire) != 0;+ else {+ while (c.lc_chk < c.n_read &&+ (c.lc_chk < c.base || fit(c, c.lc_chk).left.load(std::memory_order_acquire) == 0))+ ++c.lc_chk;+ wait = c.lc_chk <= (size_t) it.after;+ }+ if (wait) break;+ }- if (it.stage >= 0) {++ if (it.direct) {++ const size_t R = (size_t) c.ring_n, r = c.n_read % R;++ if (c.n_read >= R) { // the slot's last use has to be computed - and finished: the CPU writes it++ if (c.n_comp <= c.n_read - R) break;++ CK(cudaEventSynchronize(c.ring_free[r]));++ }++ const Layer& l = L[(size_t) it.il];++#if defined(STRATA_METAL)++ uint8_t* dst = (uint8_t*) strata::gpu::host_view(c.ring + r * c.ring_stride); // (device addresses are the GPU's: the CPU writes through the host view)++#else++ uint8_t* dst = c.ring + r * c.ring_stride;++#endif++ const x3::ExpertLayout& lay = l.lay;++ int nj = 0;++ for (int q = 0; q < 3; ++q) {++ IoQueue::Job& j = jobs[nj++];++ j = IoQueue::Job{};++ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[q]];++ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[q]];++ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[q]];++ j.off = l.mx_off[q] + (uint64_t) it.e * l.mx_stride[q];++ j.len = mx_part_bytes(l, q);++ j.dst = dst + lay.trellis[q];++ j.exact = true;++ j.left = &it.left;++ j.bad = &it.bad;++ j.mem = it.ram != nullptr ? it.ram + lay.trellis[q] : nullptr;++ j.map = pk.at(l.mx_file[q], j.off);++ j.keep = it.keep != nullptr ? it.keep + lay.trellis[q] : nullptr;++ }++ c.io->push(jobs, nj);++ } else if (it.stage >= 0) {+ const int64_t prev = c.stage_item[(size_t) it.stage];+ if (prev >= 0) {+ if ((size_t) prev >= c.n_dma) break;+ const cudaError_t q = cudaEventQuery(c.stage_ev[(size_t) it.stage]);+ if (q == cudaErrorNotReady) break;+ CK(q);+ }+ c.stage_item[(size_t) it.stage] = (int64_t) c.n_read;+ const Layer& l = L[(size_t) it.il];+ uint8_t* sb = c.stage + (size_t) it.stage * c.stage_stride;+ const int cs = copy_mode && it.ram != nullptr ? l.ram[(size_t) it.e] : -1; // (a copy's slot)+ int nj = 0;+ for (int r = 0; r < 3; ++r) {+ if (cs >= 0) { // a copy: each matrix's NH parts of rows, side by side into staging+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ for (int X = 0; X < NH; ++X) {+ IoQueue::Job& j = jobs[nj++];+ j = IoQueue::Job{};+ j.len = nb;+ j.dst = sb + (size_t) X * nb;+ j.left = &it.left;+ j.bad = &it.bad;+ j.mem = ram_part[(size_t) cs][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r];+ }+ sb += flow_region(r);+ continue;+ }+ IoQueue::Job& j = jobs[nj++];+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];+ j.off = l.mx_off[r] + (uint64_t) it.e * l.mx_stride[r] + mx_read_off(l, it.mi, r);+ j.len = mx_read_len(l, r);+ j.dst = sb;+ j.left = &it.left;+ j.bad = &it.bad;+ j.mem = it.ram != nullptr ? it.ram + l.lay.trellis[r] + mx_read_off(l, it.mi, r) : nullptr;+ j.map = pk.at(l.mx_file[r], j.off);+ j.keep = it.keep != nullptr ? it.keep + l.lay.trellis[r] : nullptr;+ sb += flow_region(r);+ }+ c.io->push(jobs, nj);+ }+ ++c.n_read;+ }+ const size_t R = (size_t) c.ring_n, dma0 = c.n_dma;+ struct Kick { // (every way out: what was queued is submitted)+ cudaStream_t s;+ const size_t &a, b;+ ~Kick() {+ if (a != b) kick(s);+ }+ } kq{c.cs, c.n_dma, dma0};+ while (c.n_dma < c.n_read && c.n_dma < c.n_comp + R) {+ Chunk::Item& it = fit(c, c.n_dma);+ if (it.left.load(std::memory_order_acquire) != 0) break;+ if (it.bad.load(std::memory_order_acquire)) {+ err = "reading layer " + std::to_string(it.il) + " expert " + std::to_string(it.e) ++ " from the model's files failed";+ return false;+ }+ const size_t r = c.n_dma % R;+ if (c.n_dma >= R) CK(cudaStreamWaitEvent(c.cs, c.ring_free[r], 0));+ const Layer& l = L[(size_t) it.il];+ uint8_t* dst = c.ring + r * c.ring_stride;+ const x3::ExpertLayout& lay = l.nm > 1 ? l.hlay : l.lay;++ if (it.direct) { // (already in the slot)++ CK(cudaEventRecord(c.ring_fill[r], c.cs));++ c.stream_bytes += (double) unit(l);++ ++c.n_dma;++ continue;++ }+ for (int p = 0; p < 2; ++p)+ CK(cudaMemcpyAsync(dst + lay.trellis[p], it.src[p], mx_read_len(l, p), cudaMemcpyHostToDevice, c.cs));+ if (l.nm > 1) { // (the down's part: in each of its rows the member's blocks)+ const int t2 = gg::expert_type(l.lay, 2);+ const size_t dr = gg::type_row_bytes(t2, g.ff), dh = gg::type_row_bytes(t2, g.ff / l.nm);+ CK(cudaMemcpy2DAsync(dst + lay.trellis[2], dh, it.src[2] + (size_t) it.mi * dh, dr, dh, (size_t) g.dim,+ cudaMemcpyHostToDevice, c.cs));+ } else+ CK(cudaMemcpyAsync(dst + lay.trellis[2], it.src[2], mx_part_bytes(l, 2), cudaMemcpyHostToDevice, c.cs));+ CK(cudaEventRecord(c.ring_fill[r], c.cs));+ if (it.stage >= 0) CK(cudaEventRecord(c.stage_ev[(size_t) it.stage], c.cs));+ c.stream_bytes += (double) unit(l);+ ++c.n_dma;+ }+ return true;+ }+ // layer L0 of chunk k routed (bounds b, the device's copy d_bounds): its streamed experts as they land, the VRAM+ // ones in shares between them (the device works while the reads run)+ bool flow_layer(Dev& d, Chunk& c, Layer& L0, int mi, int k, int T, const std::vector<int>& b, std::string& err) {+ cudaStream_t s = d.st;+ const uint8_t* vbase = member(L0, mi).vram_base; // (a member's: its part of each VRAM expert, L0's slots)+ if (c.spec_k != k || c.spec_il != L0.il) {+ err = "the prompt path's expert stream is out of step (layer " + std::to_string(L0.il) + ")";+ return false;+ }+ flow_rest(c, k, L0.il, b);+ const size_t lb = c.spec_begin, le = c.base + c.items.size(), n_lay = le - lb;+ int k2 = 0, il2 = 0;+ if (flow_next(c, k, L0.il, k2, il2)) flow_spec(c, k2, il2, fl_rows(c, k2, il2));+ const size_t R = (size_t) c.ring_n, bmax = std::max<size_t>(1, R / 2);+ int v_done = 0;+ while (c.n_comp < le) {+ // a batch's bounds are fixed by the stream alone (bmax items, never across the ring's end), never by what+ // has landed: the experts' outputs are summed batch after batch (the VRAM ones' shares between them), so+ // the sums come out the same on every run whatever the reads' timing+ const size_t i0 = c.n_comp, i1 = std::min({le, i0 + bmax, (i0 / R + 1) * R});+ if (!flow_pump(c, err)) return false;+ if (c.n_dma < i1) { // not all of the batch landed: wait for a read, or for a staging slot's copy+ const double tw = now_ms();+ kick(s);+ if (c.n_read > c.n_dma) c.io->wait_any(500);+ else {+ const Chunk::Item& it = fit(c, c.n_read);+ if (it.stage >= 0) CK(cudaEventSynchronize(c.stage_ev[(size_t) it.stage]));+ }+ c.wait_ms += now_ms() - tw;+ continue;+ }+ const int n = (int) (i1 - i0);+ int* bc = c.bc_pin + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;+ int* bcd = c.bc_dev + (size_t) (c.n_batch % (uint64_t) c.bc_n) * c.bc_w;+ ++c.n_batch;+ // (the items' virtual slots ascend and the slots between them have no rows: their bounds side by side)+ for (int m = 0; m < n; ++m) bc[m] = b[(size_t) fit(c, i0 + (size_t) m).v];+ bc[n] = b[(size_t) fit(c, i1 - 1).v + 1];+ if (bc[n] > bc[0]) {+ CK(cudaStreamWaitEvent(s, c.ring_fill[(i1 - 1) % R], 0));+ CK(cudaMemcpyAsync(bcd, bc, (size_t) (n + 1) * 4, cudaMemcpyHostToDevice, s));+ if (!run_set(d, c, L0, c.ring + (i0 % R) * c.ring_stride, 0, c.ring_stride, bcd, bc, 0, n, T, err))+ return false;+ }+ for (size_t i = i0; i < i1; ++i) CK(cudaEventRecord(c.ring_free[i % R], s));+ kick(s);+ c.n_comp = i1;+ if (!flow_pump(c, err)) return false;+ const int v_to = (int) ((int64_t) L0.n_slots * (int64_t) (c.n_comp - lb) / (int64_t) std::max<size_t>(1, n_lay));+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, v_to, T, err)) return false;+ v_done = v_to;+ }+ if (!run_set(d, c, L0, vbase, 0, unit(L0), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;+ while (c.base < c.n_comp && c.items.size() > 4096) { // (done with: read, copied and used)+ c.items.pop_front();+ ++c.base;+ }+ return true;+ }++ // the experts [i0, i1) of a sorted set (blobs at wbase + i * stride; bounds b on the host, d_bounds on the device)+ // over the chunk's rows, in pieces of at most C rows, each added into acc in pick order+ bool run_set(Dev& d, Chunk& c, const Layer& l, const uint8_t* wbase, int wb0, size_t stride, const int* d_bounds,+ const int* b, int i0, int i1, int T, std::string& err) {+ const int D = g.dim, FF = l.ffn;+ while (i0 < i1) {+ int j = i0 + 1;+ while (j < i1 && b[j + 1] - b[i0] <= c.C) ++j;+ const int a = b[i0], nrows = b[j] - a;+ int max_rows = 0;+ for (int i = i0; i < j; ++i) max_rows = std::max(max_rows, b[i + 1] - b[i]);+ if (nrows > 0) {+ x3::SetArgs sa;+ sa.wbase = wbase + (size_t) (i0 - wb0) * stride; // (wbase: set member wb0's blob)+ sa.stride = stride;+ sa.lay = l.nm > 1 ? l.hlay : l.lay;+ sa.n_exp = j - i0;+ sa.bounds = d_bounds + i0;+ sa.row0 = a;+ sa.max_rows = max_rows;++ sa.hbounds = b + i0;++ sa.total_rows = nrows;+ sa.n_embd = D;+ sa.n_ff = FF;+ sa.limit = g.lim;+ // gate / up read the layer's F16 copy of the activations (the same rounding the tiles make for their+ // shared tile): with 18-36 column blocks an expert the F32 rows were read that many times+ sa.x16 = strata::ds4::gate_up_x16_default(defaults_026()) ? c.x16 : nullptr;+ EkMark ek{};+ const bool ekp = prof_chunk;+ if (ekp) {+ std::lock_guard<std::mutex> lk(prof_mu);+ ek.e0 = ek_ev();+ ek.e1 = ek_ev();+ ek.e2 = ek_ev();+ ek.tg = gg::expert_type(sa.lay, 0);+ ek.td = gg::expert_type(sa.lay, 2);+ ek.nrows = nrows;+ ek.nexp = j - i0;+ ek.maxrows = max_rows;+ ek.ff = FF;+ cudaGetDevice(&ek.dev);+ cudaEventRecord(ek.e0, d.st);+ }+ if (sa.lay.fmt == gg::kFmtMXFP4) {+ gg::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);+ if (ekp) cudaEventRecord(ek.e1, d.st);+ gg::set_down(sa, c.h16, FF, c.y, D, d.st);+ } else {+ x3::set_gate_up(sa, c.x, D, c.row_tok + a, c.h16, FF, d.st);+ if (ekp) cudaEventRecord(ek.e1, d.st);+ x3::set_down(sa, c.h16, FF, c.y, D, d.st);+ }+ if (ekp) {+ cudaEventRecord(ek.e2, d.st);+ std::lock_guard<std::mutex> lk(prof_mu);+ ek_marks.push_back(ek);+ }+ if (dump_y != nullptr && (dump_y_layer < 0 || l.il == dump_y_layer)) {+ std::vector<int> bb(b + i0, b + j + 1);+ std::vector<uint16_t> hv((size_t) nrows * FF);+ std::vector<float> yv((size_t) nrows * D);+ CK(cudaStreamSynchronize(d.st));+ CK(cudaMemcpy(hv.data(), c.h16, hv.size() * 2, cudaMemcpyDeviceToHost));+ CK(cudaMemcpy(yv.data(), c.y, yv.size() * 4, cudaMemcpyDeviceToHost));+ std::lock_guard<std::mutex> lk(dump_mu);+ const int64_t hdr[6] = {l.il, a, nrows, j - i0, wbase == c.ring || (wbase > c.ring &&+ wbase < c.ring + (size_t) c.ring_n * c.ring_stride) ? 1 : 0, max_rows};+ std::fwrite(hdr, 8, 6, dump_y);+ std::fwrite(bb.data(), 4, bb.size(), dump_y);+ std::fwrite(hv.data(), 2, hv.size(), dump_y);+ std::fwrite(yv.data(), 4, yv.size(), dump_y);+ std::fflush(dump_y);+ }+ moe_combine(c.acc, c.y, c.pos, c.rw, T, l.k_exp, a, a + nrows, D, d.st);+ }+ i0 = j;+ }+ (void) err;+ return true;+ }++ bool load_dense(int di, std::string& err) {+ Dev& d = devs[(size_t) di];+ cudaSetDevice(d.id);+ size_t* acct = &d.weights;+ for (Layer& l0 : L)+ for (int mi = 0; mi < l0.nm; ++mi) {+ Layer& l = member(l0, mi);+ if (l.dev != di) continue;+ // (a group's member: its share of the heads (wq_b columns, the sinks), of wo_a's groups and wo_b's rows,+ // of the shared expert's FFN; the rest whole)+ const int nm = l0.nm, na = g.heads / std::max(1, l.ahn), ha = l.ah0 / std::max(1, l.ahn);+ const std::string& P = l.name;+ const std::string A = P + "attn.";+ bool ok = mat(d, A + "wq_a", l.wq_a, err, acct) &&+ (l.ahn == 0 || mat(d, A + "wq_b", l.wq_b, err, acct, ha, na, true)) &&+ mat(d, A + "wkv", l.wkv, err, acct) &&+ (l.ahn == 0 || mat(d, A + "wo_b", l.wo_b, err, acct, ha, na, false)) &&+ f32(d, A + "q_norm.weight", l.q_norm, err, acct, (size_t) g.q_lora) &&+ f32(d, A + "kv_norm.weight", l.kv_norm, err, acct, (size_t) g.hd) &&+ f32(d, A + "attn_sink", l.sink, err, acct, (size_t) g.heads) &&+ f32(d, P + "attn_norm.weight", l.attn_norm, err, acct, (size_t) g.dim) &&+ f32(d, P + "ffn_norm.weight", l.ffn_norm, err, acct, (size_t) g.dim) &&+ f32(d, P + "hc_attn_fn", l.hca_fn, err, acct) && f32(d, P + "hc_attn_scale", l.hca_scale, err, acct) &&+ f32(d, P + "hc_attn_base", l.hca_base, err, acct) && f32(d, P + "hc_ffn_fn", l.hcf_fn, err, acct) &&+ f32(d, P + "hc_ffn_scale", l.hcf_scale, err, acct) &&+ f32(d, P + "hc_ffn_base", l.hcf_base, err, acct) &&+ f32(d, P + "ffn.gate.weight", l.gate_w, err, acct, (size_t) l.n_exp * g.dim) &&+ f32(d, P + "ffn.gate.bias", l.gate_bias, err, acct, (size_t) l.n_exp) &&+ (l.mtp || img_tok < 0 || pk.get(P + "ffn.gate.bias_vl") == nullptr ||+ f32(d, P + "ffn.gate.bias_vl", l.gate_bias_vl, err, acct, (size_t) l.n_exp)) &&+ mat(d, P + "ffn.shared_experts.w1", l.s1, err, acct, mi, nm, true, l.il % nm) &&+ mat(d, P + "ffn.shared_experts.w2", l.s2, err, acct, mi, nm, false, l.il % nm) &&+ mat(d, P + "ffn.shared_experts.w3", l.s3, err, acct, mi, nm, true, l.il % nm);+ if (!ok) return false;+ l.sink += (size_t) l.ah0;+ for (int gi = 0; gi < l.ogn; ++gi)+ if (!mat(d, A + "wo_a.slice." + std::to_string(l.og0 + gi), l.wo_a[gi], err, acct)) return false;+ if (l.kv_src) {+ const std::string C = A + "compressor.";+ if (!mat(d, C + "wkv", l.c_wkv, err, acct) || !f32(d, C + "norm.weight", l.c_norm, err, acct)) return false;+ if (l.ratio > 1 && !mat(d, C + "wgate", l.c_wgate, err, acct)) return false;+ if (l.ratio > 1 && (!dalloc(d, l.c_kv, (size_t) l.ratio * g.hd, err) ||+ !dalloc(d, l.c_score, (size_t) l.ratio * g.hd, err)))+ return false;+ }+ if (l.ix_src) {+ const std::string I = A + "indexer.";+ if (!mat(d, I + "wq_b", l.i_wq_b, err, acct) || !f16raw(d, I + "weights_proj.weight", l.i_wproj, err, acct))+ return false;+ if (l.owns_k && (!mat(d, I + "wk", l.i_wk, err, acct) || !f32(d, I + "k_norm.weight", l.ik_norm, err, acct)))+ return false;+ }+ if (l.eg >= 0) {+ if (!mat(d, P + "engram.wkv", l.eg_wkv, err, acct)) return false;+ if (pk.get(P + "engram.gate.weight") != nullptr) { // a GGUF: q_weight * k_weight, folded already+ if (!f32(d, P + "engram.gate.weight", l.eg_w, err, acct, (size_t) g.hc * g.dim)) return false;+ } else {+ float *qw = nullptr, *kw = nullptr;+ if (!f32(d, P + "engram.q_weight", qw, err, acct) || !f32(d, P + "engram.k_weight", kw, err, acct))+ return false;+ std::vector<float> a((size_t) g.hc * g.dim), b(a.size());+ CK(cudaMemcpy(a.data(), qw, a.size() * 4, cudaMemcpyDeviceToHost));+ CK(cudaMemcpy(b.data(), kw, b.size() * 4, cudaMemcpyDeviceToHost));+ for (size_t i = 0; i < a.size(); ++i) a[i] *= b[i]; // only ever used as a product+ CK(cudaMemcpy(qw, a.data(), a.size() * 4, cudaMemcpyHostToDevice));+ l.eg_w = qw;+ }+ }+ if (!dalloc(d, l.win, (size_t) g.win * win_rb(), err, &d.caches)) return false;+ if (g.n_mtp > 0 && !l.mtp) {+ if (!dalloc(d, l.ring_bak, (size_t) kWin * win_rb(), err)) return false;+ if (l.kv_src && l.ratio > 1 && (!dalloc(d, l.c_snap, (size_t) 2 * l.ratio * g.hd, err) ||+ !dalloc(d, l.c_rows, (size_t) 2 * kWin * g.hd, err)))+ return false;+ }+ if (!dalloc(d, l.vram_tab, (size_t) l.n_exp, err, acct)) return false;+ }+ if (d.grp == devs[(size_t) head_dev].grp) { // the head: each member of its group a share of the columns+ const int hn = head_nm();+ if (!mat(d, "head", d.head, err, acct, d.mi, hn, true) || !f32(d, "norm.weight", d.norm_w, err, acct, (size_t) g.dim))+ return false;+ if (g.n_mtp > 0 && !f32(d, "mtp." + std::to_string(g.n_mtp - 1) + ".norm.weight", d.d_norm, err, acct, (size_t) g.dim))+ return false;+ if (!dalloc(d, d.logits_part, (size_t) kWin * ((g.vocab / 128 + hn - 1) / hn * 128), err)) return false;+ if (g.n_mtp > 0 && hn > 1) { // the markov steps over the group (mk_split): this member's rows of the head+ const Tensor* th = pk.get("mtp." + std::to_string(g.n_mtp - 1) + ".markov_head.head.weight");+ const int R = g.d_rank;+ head_cols(hn, d.mi, d.mk_v0, d.mk_vn); // (its share of the head's columns: the same rows)+ // F16 as the pack holds it, or a GGUF drafter's BF16 rounded to F16 as f16raw rounds d_mk_head (the same+ // values as the lead's whole rows); another type: no mk_split (the lead's whole rows)+ if (th != nullptr && (th->dtype == "F16" || th->dtype == "BF16") && th->bytes == (uint64_t) g.vocab * R * 2) {+ std::vector<uint16_t> rows((size_t) d.mk_vn * R);+ if (!pk.read_at(th->file, th->off + (uint64_t) d.mk_v0 * R * 2, rows.size() * 2, rows.data())) {+ err = "the markov head: read failed";+ return false;+ }+ if (th->dtype == "BF16")+ for (uint16_t& h : rows) h = strata::kernels::f16_from_f32(bf16_to_f32(h));+ if (!dalloc(d, d.mk_share, rows.size(), err, acct) || !dalloc(d, d.mk_erow, (size_t) R, err) ||+ !dalloc(d, d.mk_ints, (size_t) kWin + 2, err))+ return false;+ CK(cudaMemcpy(d.mk_share, rows.data(), rows.size() * 2, cudaMemcpyHostToDevice));+ }+ }+ }+ if (di == head_dev) {+ head = d.head;+ norm_w = d.norm_w;+ d_norm = d.d_norm;+ if (g.n_mtp > 0) {+ const std::string Z = "mtp." + std::to_string(g.n_mtp - 1) + ".";+ if (!f32(d, Z + "confidence_head.proj.weight", d_conf, err, acct, (size_t) (g.dim + g.d_rank)))+ return false;+ const Tensor* te = pk.get(Z + "markov_head.embed.weight");+ const Tensor* th = pk.get(Z + "markov_head.head.weight");+ if (te == nullptr || th == nullptr || te->dtype != "BF16" || (th->dtype != "F16" && th->dtype != "BF16")) {+ err = Z + "markov_head: not the BF16 embed and F16 / BF16 head expected";+ return false;+ }+ if (!dalloc(d, d_mk_embed, te->bytes / 2, err, acct) || !upload(d, *te, d_mk_embed, err) ||+ (strata::ds4::markov_head_q8_default(defaults_026())+ ? !mk_head_q8(d, Z + "markov_head.head.weight", d_mk_head_q8, err, acct)+ : !f16raw(d, Z + "markov_head.head.weight", d_mk_head, err, acct))) // (0.2.6: the head as F16)+ return false;+ }+ }+ if (g.n_mtp > 0 && di == d_dev) {+ if (!mat(d, "mtp.0.main_proj", d_main_proj, err, acct) ||+ !f32(d, "mtp.0.main_norm.weight", d_main_norm, err, acct, (size_t) g.dim))+ return false;+ }+ return true;+ }+++#if defined(STRATA_METAL)++ // The most the VRAM experts may add on a Mac: the engine's anonymous memory (Metal buffers + host memory, all counted) stays++ // under the cap metal/src/engine/arena_budget.h derived from the freezes of the 128 GB M5 Max (macOS keeps a file-cache floor and++ // compresses / swaps anonymous memory past it): max(15% of the RAM, 40% of the RAM, the floor + 12 GB) left to macOS, at most++ // the GPU working set; what the process holds now (task_info phys_footprint) is already inside it.++ double metal_expert_cap() const {++ uint64_t phys = 0;++ size_t len = sizeof phys;++ sysctlbyname("hw.memsize", &phys, &len, nullptr, 0);++ unsigned long long pages = 0;++ size_t pl = sizeof pages;++ double floor_b = 0;++ if (sysctlbyname("vm.vm_page_filecache_min", &pages, &pl, nullptr, 0) == 0) floor_b = (double) pages * (double) vm_page_size;++ size_t fr = 0, tot = 0;++ cudaMemGetInfo(&fr, &tot);++ const double gb = 1e9, P = (double) phys;++ double keep = std::max(7.0 * gb, 0.15 * P);++ if (const char* e = std::getenv("STRATA_METAL_KEEP_FREE_GB")) keep = std::atof(e) * gb;++ double share = 0.40;++ if (const char* e = std::getenv("STRATA_METAL_LEAVE_SHARE")) share = std::atof(e);++ double leave = keep;++ if (P >= 64.0 * gb) leave = std::max({leave, share * P, floor_b > 0 ? floor_b + 12.0 * gb : 0.0});++ const double cap = std::min((double) tot, P - leave);++ task_vm_info_data_t ti;++ mach_msg_type_number_t cnt = TASK_VM_INFO_COUNT;++ double foot = 0;++ if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &ti, &cnt) == KERN_SUCCESS) foot = (double) ti.phys_footprint;++ return std::max(4.0 * gb, cap - foot - 1.0 * gb);++ }++#endif++#if defined(STRATA_METAL)++ // Metal (unified memory): a CPU-lane expert's matrices through no-copy buffers over the model's own pages, so the GPU computes it++ // like a VRAM one (no CPU lane, no second pass). false: not mappable / the cap is reached (the CPU lane has it).++ bool zc_try(Layer& l, int e) {++ static const bool on = std::getenv("DS4_ZC") != nullptr; // (measured: no faster than the CPU lane - cold pages come from the SSD either way)++ if (!on || l.lay.fmt != gg::kFmtMXFP4 || l.nm != 1 || l.vm.empty()) return false;++ if (l.vm[(size_t) e][0].raw != nullptr) return true;++ uint64_t a[3];++ for (int r = 0; r < 3; ++r) {++ a[r] = ::strata::dsv41::metal_host::zc_map(mx_mat(l, e, r), mx_part_bytes(l, r));++ if (a[r] == 0) return false;++ }++ for (int r = 0; r < 3; ++r) {++ x3::Mat& m = l.vm[(size_t) e][(size_t) r];++ m = x3::Mat{};++ m.K = gg::expert_type(l.lay, r);++ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;++ m.raw = (const void*) a[r];++ m.k = r < 2 ? g.dim : g.ff;++ m.n = r < 2 ? g.ff : g.dim;++ }++ return true;++ }++#endif+ void set_vm(Layer& l, int e, int slot) {+ uint8_t* b = l.vram_base + (size_t) slot * l.blob;+ if (l.lay.fmt == gg::kFmtMXFP4) {+ for (int r = 0; r < 3; ++r) {+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];+ m = x3::Mat{};+ m.K = gg::expert_type(l.lay, r); // (a low-bit type: its GEMV, ds4_iq's)+ m.fmt = m.K == gg::kTypeMXFP4 ? gg::kFmtMXFP4 : gg::kFmtGG;+ m.raw = b + l.lay.trellis[r];+ m.k = r < 2 ? g.dim : g.ff;+ m.n = r < 2 ? g.ff : g.dim;+ }+ return;+ }+ for (int r = 0; r < 3; ++r) {+ x3::Mat& m = l.vm[(size_t) e][(size_t) r];+ m.trellis = (const uint32_t*) (b + l.lay.trellis[r]);+ m.suh = (const uint16_t*) (b + l.lay.suh[r]);+ m.svh = (const uint16_t*) (b + l.lay.svh[r]);+ m.k = r < 2 ? g.dim : g.ff;+ m.n = r < 2 ? g.ff : g.dim;+ m.K = l.lay.K[r];+ m.cb = 2;+ m.ld = 0;+ }+ }+ // an expert's VRAM blob -> its CPU-lane part X, and its NH parts -> the blob (the cache's swaps)+ void blob_to_part(const Layer& l, const uint8_t* blob, int X, uint8_t* dst) const {+ for (int r = 0; r < 3; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;+ const uint8_t* tr = blob + l.lay.trellis[r];+ for (int sl = 0; sl < k / 16; ++sl)+ std::memcpy(dst + l.hl.trellis[r] + (size_t) sl * nth * tile, tr + ((size_t) sl * nt + (size_t) X * nth) * tile,+ (size_t) nth * tile);+ std::memcpy(dst + l.hl.suh[r], blob + l.lay.suh[r], (size_t) k * 2);+ std::memcpy(dst + l.hl.svh[r], blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, (size_t) l.hl.n[r] * 2);+ }+ }+ void parts_to_blob(const Layer& l, const std::array<uint8_t*, 4>& parts, uint8_t* blob) const {+ for (int r = 0; r < 3; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r];+ const int k = l.hl.k[r], nt = l.hl.n[r] * NH / 16, nth = l.hl.n[r] / 16;+ uint8_t* tr = blob + l.lay.trellis[r];+ for (int X = 0; X < NH; ++X)+ for (int sl = 0; sl < k / 16; ++sl)+ std::memcpy(tr + ((size_t) sl * nt + (size_t) X * nth) * tile,+ parts[(size_t) X] + l.hl.trellis[r] + (size_t) sl * nth * tile, (size_t) nth * tile);+ std::memcpy(blob + l.lay.suh[r], parts[0] + l.hl.suh[r], (size_t) k * 2);+ for (int X = 0; X < NH; ++X)+ std::memcpy(blob + l.lay.svh[r] + (size_t) X * l.hl.n[r] * 2, parts[(size_t) X] + l.hl.svh[r],+ (size_t) l.hl.n[r] * 2);+ }+ }++ // an expert's blob -> member m's part (gate / up columns [ff0, ff0 + ffn), down rows of the same), and back+ void blob_to_half(const Layer& l, const uint8_t* blob, int m, uint8_t* dst) const {+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_blob_to_half(l, blob, m, dst);+ const Layer& x = member(l, m);+ const int f0 = x.ff0, fn = x.ffn;+ for (int r = 0; r < 2; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)+ std::memcpy(dst + x.hlay.trellis[r] + sl * nth * tile,+ blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile, nth * tile);+ std::memcpy(dst + x.hlay.suh[r], blob + l.lay.suh[r], (size_t) g.dim * 2);+ std::memcpy(dst + x.hlay.svh[r], blob + l.lay.svh[r] + (size_t) f0 * 2, (size_t) fn * 2);+ }+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];+ std::memcpy(dst + x.hlay.trellis[2], blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, (size_t) (fn / 16) * row);+ std::memcpy(dst + x.hlay.suh[2], blob + l.lay.suh[2] + (size_t) f0 * 2, (size_t) fn * 2);+ std::memcpy(dst + x.hlay.svh[2], blob + l.lay.svh[2], (size_t) g.dim * 2);+ }+ void half_to_blob(const Layer& l, const uint8_t* src, int m, uint8_t* blob) const {+ if (l.lay.fmt == gg::kFmtMXFP4) return mx_half_to_blob(l, src, m, blob);+ const Layer& x = member(l, m);+ const int f0 = x.ff0, fn = x.ffn;+ for (int r = 0; r < 2; ++r) {+ const size_t tile = (size_t) 32 * l.lay.K[r], nt = (size_t) g.ff / 16, nth = (size_t) fn / 16;+ for (size_t sl = 0; sl < (size_t) g.dim / 16; ++sl)+ std::memcpy(blob + l.lay.trellis[r] + (sl * nt + (size_t) f0 / 16) * tile,+ src + x.hlay.trellis[r] + sl * nth * tile, nth * tile);+ if (m == 0) std::memcpy(blob + l.lay.suh[r], src + x.hlay.suh[r], (size_t) g.dim * 2);+ std::memcpy(blob + l.lay.svh[r] + (size_t) f0 * 2, src + x.hlay.svh[r], (size_t) fn * 2);+ }+ const size_t row = (size_t) (g.dim / 16) * 32 * l.lay.K[2];+ std::memcpy(blob + l.lay.trellis[2] + (size_t) (f0 / 16) * row, src + x.hlay.trellis[2], (size_t) (fn / 16) * row);+ std::memcpy(blob + l.lay.suh[2] + (size_t) f0 * 2, src + x.hlay.suh[2], (size_t) fn * 2);+ if (m == 0) std::memcpy(blob + l.lay.svh[2], src + x.hlay.svh[2], (size_t) g.dim * 2);+ }++ bool place_experts(std::string& err) {+ // (Windows, an integrated GPU: what its experts may commit - commit_room - and what that kept off the GPU)+ double room = 1e300, kept = 0;+ for (const Dev& d : devs)+ if (commit_charged(d)) room = commit_room();+ const double room0 = room;+ // per group: what its tightest member has left (each member holds its part of every VRAM expert of the+ // group's layers, its unit bytes a slot - counted at the members' mean: an uneven split's odd blocks go round+ // the members layer by layer, so each member's total comes to about the mean's)+ auto ucount = [](const Layer& l) {+ double s = 0;+ for (int m = 0; m < l.nm; ++m) s += (double) unit(member(l, m));+ return s / l.nm;+ };+ for (size_t gi = 0; gi < groups.size(); ++gi) {+ double budget = 1e300;+ bool charged = false;+ for (int m : groups[gi].mem) {+ cudaSetDevice(devs[(size_t) m].id);+ size_t fr = 0, tot = 0;+ CK(cudaMemGetInfo(&fr, &tot));+ // (an integrated GPU, Windows: its memory is the system's. What it allocates after the experts - the long+ // prompt's buffers, the GEMM workspaces - spills into RAM when the experts took every byte of the GPU's+ // address space, and on a 32K prompt that ran the machine out of RAM. So the experts leave room for them,+ // growing with the context. DS4_UMA_GPU_RESERVE_GB: that room, tests)+ double uma_extra = 0.0;+ if (commit_charged(devs[(size_t) m])) {+ uma_extra = (4.0 + 2.0e-4 * std::max(0.0, (double) o.max_context - 16384.0)) * (double) (1ull << 30);+ if (const char* v = std::getenv("DS4_UMA_GPU_RESERVE_GB")) uma_extra = std::atof(v) * (double) (1ull << 30);+ }+ budget = std::min(budget, std::max(0.0, ((double) fr - (double) ((size_t) o.reserve_mib << 20) - uma_extra)) *+ o.vram_experts);++#if defined(STRATA_METAL)++ if (o.vram_experts_auto && std::getenv("DS4_NO_AUTO_VX") == nullptr) {++ const double cap = metal_expert_cap();++ if (cap < budget) {++ std::fprintf(stderr, "ds4: expert share: %.1f GB of VRAM experts (the RAM cap leaves %.1f GB; --vram-experts sets it)\n",++ cap / 1e9, cap / 1e9);++ budget = cap;++ }++ }++#endif+ charged = charged || commit_charged(devs[(size_t) m]);+ }+ if (charged && room < 1e300) { // (each member's slots charged: the group's share of the room)+ double all = 0;+ for (const Layer& l : L)+ if (l.grp == (int) gi) all += (double) unit(l) * l.n_exp;+ const double nm = (double) groups[gi].mem.size(), capped = std::min(budget, room / nm);+ kept += std::max(0.0, std::min(budget, all) - std::min(capped, all)) * nm;+ budget = capped;+ room = std::max(0.0, room - std::min(budget, all) * nm);+ }+ double budget_t = budget; // the drafter's experts first, all of them (every draft uses its blocks)+ for (Layer& l : L)+ if (l.grp == (int) gi && l.mtp) {+ l.n_vram = std::min(l.n_exp, (int) std::floor(budget_t / (double) ucount(l)));+ budget_t -= (double) l.n_vram * ucount(l);+ }+ double want = 0;+ for (const Layer& l : L)+ if (l.grp == (int) gi && !l.mtp && !l.cpu_only) want += (double) ucount(l) * l.n_exp;+ const double frac = want > 0 ? std::min(1.0, std::max(0.0, budget_t) / want) : 0.0;+ for (Layer& l : L) {+ if (l.grp != (int) gi || l.mtp) continue;+ if (l.cpu_only) { // (its type has no device kernel: the CPU lane reads every expert of it)+ l.n_vram = 0;+ continue;+ }+ l.n_vram = std::min(l.n_exp, (int) std::floor(frac * l.n_exp));+ // never more than fits: the floor of each layer's share can still round up the total+ while (l.n_vram > 0 && (double) ucount(l) * l.n_vram > budget) --l.n_vram;+ }+ double used = 0;+ for (Layer& l : L)+ if (l.grp == (int) gi) used += (double) ucount(l) * l.n_vram;+ // what the floors left: whole experts, layer by layer+ for (Layer& l : L) {+ if (l.grp != (int) gi || l.cpu_only) continue;+ while (l.n_vram < l.n_exp && used + (double) ucount(l) <= budget) {+ ++l.n_vram;+ used += (double) ucount(l);+ }+ }+ }+ if (kept > 0) { // the commit kept experts off the GPU: say so, and the page file that keeps them on it+ const os::Commit c = os::commit();+ const double gib = (double) (1ull << 30);+ const double pf = c.limit != ~uint64_t{0} && c.ram != ~uint64_t{0} && c.limit > c.ram ? (double) (c.limit - c.ram) : 0.0;+ commit_kept = kept;+ pagefile_hint = std::ceil((pf + kept) / gib);+ std::fprintf(stderr,+ "ds4: WARNING: %.1f GB of experts stay off the GPU - Windows charges the GPU's memory to its "+ "commit (RAM + page file) and can commit only %.1f GB more for them (%.0f GB kept for the "+ "other programs). A page file of %.0f GB (%.0f MB; now %.0f GB) keeps them on the GPU: System "+ "> About > Advanced system settings > Performance > Advanced > Virtual memory, then restart\n",+ kept / 1e9, room0 / 1e9, commit_margin() / 1e9, pagefile_hint, pagefile_hint * 1024.0, pf / gib);+ } else if (room0 < 1e300 && o.verbose) {+ std::fprintf(stderr, "ds4: Windows' commit has room for the GPU's experts (%.1f GB beside %.0f GB for the other "+ "programs)\n", room0 / 1e9, commit_margin() / 1e9);+ }+ for (Layer& l : L) {+ // the hottest experts of the profile first, then by index+ std::vector<int> order((size_t) l.n_exp);+ std::iota(order.begin(), order.end(), 0);+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ std::stable_sort(order.begin(), order.end(), [&](int a, int b) { return p[a] > p[b]; });+ l.slot.assign((size_t) l.n_exp, -1);+ l.host.assign((size_t) l.n_exp, nullptr);+ // a layer with experts on both sides keeps one VRAM slot free for the cache's swaps+ l.n_slots = std::min(l.n_vram, l.n_exp);+ if (o.expert_cache && l.n_slots >= 2 && l.n_slots < l.n_exp) { // (every expert fits: nothing to swap)+ l.n_vram = l.n_slots - 1;+ l.spare_vram = l.n_slots - 1;+ } else {+ l.n_vram = l.n_slots;+ l.spare_vram = -1;+ }+ for (int i = 0; i < l.n_vram; ++i) l.slot[(size_t) order[(size_t) i]] = i;+ for (int m = 0; m < l.nm && l.n_slots > 0; ++m) {+ Layer& x = member(l, m);+ Dev& d = devs[(size_t) x.dev];+ cudaSetDevice(d.id);+ void* v = nullptr;+ if (cudaMalloc(&v, (size_t) l.n_slots * unit(x)) != cudaSuccess) {+ err = "cudaMalloc of layer " + std::to_string(l.il) + "'s " + std::to_string(l.n_slots) ++ " experts on CUDA" + std::to_string(d.id) + " failed";+ return false;+ }+ d.allocs.push_back(v);+ d.experts += (size_t) l.n_slots * unit(x);+ x.vram_base = (uint8_t*) v;+ }+ l.heat.assign((size_t) l.n_exp, 0.0f);+ { // the profile as the starting heat: its counts scaled to ~512 hits a layer+ double sum = 0;+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];+ const double sc = sum > 0 ? 512.0 / sum : 0.0;+ for (int e = 0; e < l.n_exp; ++e) l.heat[(size_t) e] = (float) (p[e] * sc);+ }+ const int n_host = l.n_exp - l.n_vram;+ { // the CPU lane's parts of an expert (in place: the whole expert, its pieces where the file has them)+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ const int k = r < 2 ? g.dim : g.ff, nf = r < 2 ? g.ff : g.dim, n = nf / NH;+ l.hl.k[r] = k;+ l.hl.n[r] = n;+ l.hl.trellis[r] = file_experts ? l.piece_src[r][0] : off;+ off += align256((size_t) (k / 16) * (n / 16) * 32 * l.lay.K[r]);+ l.hl.suh[r] = file_experts ? l.piece_src[r][1] : off;+ off += align256((size_t) k * 2);+ l.hl.svh[r] = file_experts ? l.piece_src[r][2] : off;+ off += align256((size_t) n * 2);+ }+ l.hl.bytes = off;+ }+ l.hh.assign((size_t) l.n_exp, {nullptr, nullptr, nullptr, nullptr});+ if (l.lay.fmt == gg::kFmtMXFP4) { // a GGUF's: read in place from the file's mapping+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = (uint8_t*) mx_mat(l, e, 0);+ } else if (file_experts) { // every expert's file mapped (the cache's swaps hand VRAM experts back to the files)+ for (int e = 0; e < l.n_exp; ++e)+ if (pk.map(l.region_file[(size_t) e]) == nullptr) {+ err = "mapping " + pk.files[(size_t) l.region_file[(size_t) e]] + " failed";+ return false;+ }+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) {+ l.hh[(size_t) e][0] = file_part(l, e);+ l.host[(size_t) e] = l.hh[(size_t) e][0];+ }+ } else if (n_host > 0) {+ const int n_parts = n_host + (l.spare_vram >= 0 ? 1 : 0); // (+ the cache's spare)+ const size_t bytes = (size_t) n_parts * l.hl.bytes;+ for (int X = 0; X < NH; ++X) {+ void* m = node_map(bytes, X, NH);+ if (m == nullptr) {+ err = "host memory for layer " + std::to_string(l.il) + "'s " + std::to_string(n_host) ++ " CPU experts (" + std::to_string(bytes >> 20) + " MiB on node " + std::to_string(X) + ")";+ return false;+ }+ host_maps.emplace_back(m, bytes);+ int hi = 0;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) l.hh[(size_t) e][(size_t) X] = (uint8_t*) m + (size_t) (hi++) * l.hl.bytes;+ if (l.spare_vram >= 0) l.spare_host[(size_t) X] = (uint8_t*) m + (size_t) hi * l.hl.bytes;+ }+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) l.host[(size_t) e] = l.hh[(size_t) e][0];+ }+ l.vm.assign((size_t) l.n_exp, {});+ if (l.nm == 1) // (the host-routed window path's views: a group's members take the device-routed path)+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] >= 0) set_vm(l, e, l.slot[(size_t) e]);+ if (!halloc(l.tab_pin, (size_t) 2 * l.nm, err)) return false;+ for (int m = 0; m < l.nm; ++m) {+ Layer& x = member(l, m);+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] >= 0)+ tab[(size_t) e] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) e] * unit(x));+ cudaSetDevice(devs[(size_t) x.dev].id);+ CK(cudaMemcpy(x.vram_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice));+ }+ }+ return true;+ }++ // a GGUF's VRAM experts, in file order, by a few threads through pinned staging blobs (the CPU lane's stay in the+ // file: the mapping reads them on first use)+ bool load_experts_mx(std::string& err) {+ struct Job {+ int layer, e;+ };+ std::vector<Job> jobs;+ size_t maxb = 0;+ for (const Layer& l : L) {+ maxb = std::max(maxb, l.blob);+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});+ }+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];+ });+ std::atomic<size_t> next{0};+ std::atomic<bool> failed{false};+ std::mutex emu;+ std::string ferr;+ std::atomic<int64_t> done_bytes{0};+ const double t0 = now_ms();+ std::vector<std::thread> th;+ for (int t = 0; t < 12; ++t)+ th.emplace_back([&] {+ uint8_t *stage = nullptr, *stage2 = nullptr;+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);+ for (;;) {+ const size_t i = next.fetch_add(1);+ if (i >= jobs.size() || failed.load()) break;+ const Layer& l = L[(size_t) jobs[i].layer];+ const int e = jobs[i].e;+ bool ok = stage != nullptr && stage2 != nullptr && read_mx(l, e, stage);+ for (int m = 0; m < l.nm && ok; ++m) { // (a group: each member its part)+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);+ ok = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(l), l.nm > 1 ? stage2 : stage,+ unit(l), cudaMemcpyHostToDevice) == cudaSuccess;+ }+ if (!ok) {+ std::lock_guard<std::mutex> lk(emu);+ ferr = "loading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";+ failed.store(true);+ break;+ }+ const int64_t db = done_bytes.fetch_add((int64_t) l.blob) + (int64_t) l.blob;+ if (o.verbose && (i % 1024) == 0)+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,+ db / 1e6 / std::max(1.0, now_ms() - t0));+ }+ if (stage) cudaFreeHost(stage);+ if (stage2) cudaFreeHost(stage2);+ });+ for (auto& t : th) t.join();+ if (failed.load()) {+ err = ferr;+ return false;+ }+ // as load_experts in place: the VRAM experts' pages leave the cache, the most-routed of the others (the+ // profile's share of their layer, then by index) are read once, up to warm_bytes, so the cache starts with them+ struct Warm {+ int layer, e;+ double share;+ };+ std::vector<Warm> warm;+ for (const Layer& l : L) {+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ double sum = 0;+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];+ for (int e = 0; e < l.n_exp; ++e) {+ if (l.slot[(size_t) e] >= 0) {+ for (int r = 0; r < 3; ++r)+ os::advise(pk.fds[(size_t) l.mx_file[r]], (l.mx_off[r] + (uint64_t) e * l.mx_stride[r]), mx_part_bytes(l, r), false);+ } else {+ warm.push_back({l.il, e, sum > 0 ? p[e] / sum : 0.0});+ }+ }+ }+ std::stable_sort(warm.begin(), warm.end(), [](const Warm& a, const Warm& b) { return a.share > b.share; });+ if (!ram_tier && !warm.empty()) { // copied into RAM (copy_experts decides), the rest read ahead into what is left+ std::vector<std::pair<int, int>> hot;+ for (const Warm& w : warm) hot.push_back({w.layer, w.e});+ if (!copy_experts(hot, err)) return false;+ if (copy_mode) {+ warm.erase(std::remove_if(warm.begin(), warm.end(),+ [&](const Warm& w) { return L[(size_t) w.layer].ram[(size_t) w.e] >= 0; }),+ warm.end());+ const uint64_t av = ram_avail();+ if (av != ~uint64_t{0}) warm_bytes = av > kRamMargin ? std::min<uint64_t>(warm_bytes, av - kRamMargin) : 0;+ }+ }+ if (ram_tier && warm_bytes > 0 && !warm.empty()) {+ // chunks of 32 slots up to warm_bytes (all the RAM the rest leaves, kRamMargin kept), one slot kept free for+ // a move in; the hottest experts read in, their pages then dropped from the page cache. Plain memory, not+ // pinned: only the CPU reads it (what goes to a GPU passes the pinned staging), and pinned memory is capped on+ // Windows - the engine's own pinned buffers come after this+ const size_t want = std::min(warm.size(), (size_t) (warm_bytes / maxb));+ while (ram_slots.size() < want + 1) {+ const size_t n = std::min<size_t>(32, want + 1 - ram_slots.size());+ uint8_t* c = (uint8_t*) os::anon_alloc(n * maxb, -1, 1);+ if (c == nullptr) break;+ ram_chunks.push_back(c);+ ram_chunk_bytes.push_back(n * maxb);+ for (size_t i = 0; i < n; ++i) ram_slots.push_back(c + i * maxb);+ }+ ram_held = ram_slots.empty() ? 0 : (int) ram_slots.size() - 1;+ for (Layer& l : L)+ if (l.lay.fmt == gg::kFmtMXFP4) l.ram.assign((size_t) l.n_exp, -1);+ std::atomic<size_t> ri{0};+ std::atomic<bool> rfail{false};+ std::vector<std::thread> rt;+ for (int t = 0; t < 12 && ram_held > 0; ++t)+ rt.emplace_back([&] {+ for (size_t i; (i = ri.fetch_add(1)) < (size_t) ram_held;) {+ const Layer& l = L[(size_t) warm[i].layer];+ if (!read_mx_file(l, warm[i].e, ram_slots[i])) rfail.store(true);+ for (int r = 0; r < 3; ++r)+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) warm[i].e * l.mx_stride[r],+ mx_part_bytes(l, r), false);+ }+ });+ for (auto& t : rt) t.join();+ if (rfail.load()) {+ err = "reading the RAM tier's experts from the model's files failed";+ return false;+ }+ for (int i = 0; i < ram_held; ++i) L[(size_t) warm[(size_t) i].layer].ram[(size_t) warm[(size_t) i].e] = i;+ if (ram_held > 0) ram_free.push_back(ram_held);+ if (o.expert_cache && ram_held > 0) ram_th = std::thread([this] { ram_loop(); });+ if (o.verbose)+ std::fprintf(stderr, "ds4: RAM tier: %d experts (%.1f GB of RAM), the hottest after VRAM's; the other %zu "+ "read from the model's files (%.1f s)\n",+ ram_held, ram_slots.size() * (double) maxb / 1e9, warm.size() - (size_t) ram_held,+ (now_ms() - t0) / 1000.0);+ return true;+ }+ size_t nw = 0;+ for (double b = 0; nw < warm.size() && b + (double) L[(size_t) warm[nw].layer].blob <= (double) warm_bytes; ++nw)+ b += (double) L[(size_t) warm[nw].layer].blob;+ std::atomic<size_t> wi{0};+ std::vector<std::thread> wt;+ for (int t = 0; t < 12 && nw > 0; ++t)+ wt.emplace_back([&] {+ std::vector<uint8_t> buf(maxb);+ for (size_t i; (i = wi.fetch_add(1)) < nw;) read_mx(L[(size_t) warm[i].layer], warm[i].e, buf.data());+ });+ for (auto& t : wt) t.join();+ if (o.verbose && nw > 0)+ std::fprintf(stderr, "ds4: %zu CPU-lane experts read ahead (%.1f GB) in %.1f s\n", nw,+ nw * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0);+ return true;+ }++ // a GGUF's CPU-lane experts copied into RAM (copy_mode), decided once VRAM's are in - on a unified-memory APU the GPU+ // has taken its share of the RAM by then, so the copies only get what it leaves. `hot`: the CPU lane's experts, the+ // most-routed first. --mmap-experts off: all that fit beside kRamMargin (a warning if not all do); on and auto:+ // none - read in place: the page cache follows a conversation's experts where a static set (the profile's+ // hottest) does not (the RAM tier's lesson), and copies that all fit were no faster; DS4_COPY_GB=<n>: the+ // hottest n GB (the static hybrid, for tests). Each copy in NH parts on the NUMA nodes (DS4_COPY_NUMA=1: one), read from the files in file+ // order by a few threads, anonymous memory on huge pages (Windows: plain), not pinned (only the CPU reads it; what+ // goes to a GPU passes the pinned staging) - a layer that cannot get its memory keeps its experts in place+ bool copy_experts(const std::vector<std::pair<int, int>>& hot, std::string& err) {+ if (o.mmap_experts > 0 || hot.empty()) return true;+ const double t0 = now_ms();+ auto spare_of = [&](const Layer& l) { return o.expert_cache && l.spare_vram >= 0 ? 1 : 0; };+ double all = 0;+ {+ std::vector<char> seen(L.size(), 0);+ for (const auto& h : hot) {+ const Layer& l = L[(size_t) h.first];+ all += (double) l.blob * (1 + (seen[(size_t) l.il]++ == 0 ? spare_of(l) : 0));+ }+ }+ const uint64_t avail = ram_avail();+ // (Windows: the copies are committed memory, within what its commit still has beside the other programs' margin)+ // DS4_COPY_HUGETLB: the arena comes from the kernel's huge-page pool, which MemAvailable does not count: add+ // its free bytes so the full copy is taken when the pool holds it.+ // 0.2.7: DS4_COPY_HUGETLB unset = ON when the reserved pool already holds the whole copy (HugePages_Free), else the+ // ordinary arena exactly as before; =0 is the off switch, =1 asks for it anyway (a smaller pool is used, the rest warns).+ const uint64_t pool_free = os::huge_free_bytes();+ const bool huge_arena = strata::ds4::copy_hugetlb_default(env_tri("DS4_COPY_HUGETLB"), (double) pool_free, all, defaults_026());+ if (huge_arena && std::getenv("DS4_COPY_HUGETLB") == nullptr)+ std::fprintf(stderr, "ds4: the reserved huge-page pool (%.1f GB free) holds the whole copy (%.1f GB): the arena on 2 MB pages "+ "(DS4_COPY_HUGETLB=0 keeps the ordinary arena)\n", (double) pool_free / 1e9, all / 1e9);+ bool arena_all_huge = huge_arena; // (every chunk really from the pool: the pin below needs that)+ const double pool = huge_arena ? (double) pool_free : 0.0;+ const double can = std::min(avail == ~uint64_t{0} ? 1e300 : std::max(0.0, (double) avail - (double) kRamMargin) + pool,+ commit_room());+ // (auto: none - copies that all fit were measured no faster than the page cache: the CPU lane's rows are+ // compute-bound, ~46 GB/s from either on 2 Xeon sockets, NUMA parts included; a part copied cost decode speed+ // on 64 GB and on 32 GB of RAM)+ double budget = o.mmap_experts == 0 ? std::min(all, can) : auto_copy_all && all <= can ? all : 0.0; // (auto: all or none)+ if (const char* v = std::getenv("DS4_COPY_GB")) budget = std::atof(v) * 1e9;+ if (o.mmap_experts < 0 && auto_copy_all && all > can)+ std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) do not all fit the RAM this process can get now (%.1f GB beside %.0f GB for "+ "the rest): read in place from the model's files (--mmap-experts off copies what fits)\n", all / 1e9, can / 1e9, kRamMargin / 1e9);+ if (o.mmap_experts == 0 && all > can)+ std::fprintf(stderr, "ds4: --mmap-experts off: the CPU lane's experts (%.1f GB) do not all fit the RAM this "+ "process can get (%.1f GB beside %.0f GB for the rest) - the most-routed copied, the "+ "others read in place\n", all / 1e9, can / 1e9, kRamMargin / 1e9);+ if (auto_copy_all && budget > 0)+ std::fprintf(stderr, "ds4: --mmap-experts auto: the CPU lane's experts (%.1f GB) fit the RAM this process can get (%.1f GB beside %.0f GB for "+ "the rest): copied in at load, no demand store (--mmap-experts on keeps them in place)\n", all / 1e9, can / 1e9, kRamMargin / 1e9);+ if (budget <= 0) {+ if (o.verbose)+ std::fprintf(stderr, "ds4: the CPU lane's experts (%.1f GB) stay in place: the RAM this process can get "+ "(%.1f GB beside %.0f GB for the rest) does not hold them all\n",+ all / 1e9, can / 1e9, kRamMargin / 1e9);+ return true;+ }+ // NUMA parts: whole rows of each matrix, so any row count the nodes divide+ int nh = (int) std::min<size_t>(2, std::max<size_t>(1, cores.size()));+ if (const char* v = std::getenv("DS4_COPY_NUMA")) // (0 / 1: one part; 2: two, even on one node - a test)+ nh = std::atoi(v) >= 2 ? 2 : 1;+ if (g.ff % nh != 0 || g.dim % nh != 0) nh = 1;+ NH = nh;+ cp_lay.assign(L.size(), {0, 0, 0, 0});+ for (const Layer& l : L) {+ size_t off = 0;+ for (int r = 0; r < 3; ++r) {+ cp_lay[(size_t) l.il][(size_t) r] = off;+ off += align256((size_t) (mx_n(r) / NH) * mx_row_b(l, r));+ }+ cp_lay[(size_t) l.il][3] = off;+ }+ // the hottest that fit the budget (a layer's first one brings its spare)+ std::vector<int> n_copy(L.size(), 0);+ double used = 0;+ size_t take = 0;+ for (; take < hot.size(); ++take) {+ const Layer& l = L[(size_t) hot[take].first];+ const double add = (double) l.blob * (1 + (n_copy[(size_t) l.il] == 0 ? spare_of(l) : 0));+ if (used + add > budget) break;+ used += add;+ ++n_copy[(size_t) l.il];+ }+ // each layer's slots: one block per node of (copies + spare) parts+ std::vector<int> first(L.size(), -1);+ copy_spare.assign(L.size(), -1);+ for (Layer& l : L) {+ l.ram.assign((size_t) l.n_exp, -1);+ const int ns = n_copy[(size_t) l.il] + (n_copy[(size_t) l.il] > 0 ? spare_of(l) : 0);+ if (ns == 0) continue;+ const size_t part = cp_lay[(size_t) l.il][3], bytes = (size_t) ns * part;+ std::array<uint8_t*, 2> base = {nullptr, nullptr};+ // DS4_COPY_HUGETLB=1: the arena on explicit 2 MB pages (the pool must be reserved), so a pin of it keeps+ // 2 MB pages and the prompt's DMA reads stay stable (a THP arena pinned by cudaHostRegister runs in 4 KB+ // pages: measured 11.9 s one prompt, 30.4 s the next for the same 140 GB)+ const bool huge = huge_arena;+ bool ok = true;+ for (int X = 0; X < NH && ok; ++X) {+ size_t got = bytes;+ base[(size_t) X] = (uint8_t*) (huge ? os::huge_alloc(bytes, &got) : node_map(bytes, X, NH));+ if (huge && base[(size_t) X] == nullptr) { // no pool on this machine: the ordinary arena instead+ static bool warned = false;+ if (!warned) {+ std::fprintf(stderr, "ds4: the 2 MB huge-page arena was asked for but the pool ran out (reserve more: sysctl "+ "vm.nr_hugepages=N); using the ordinary arena\n");+ warned = true;+ }+ arena_all_huge = false;+ got = bytes;+ base[(size_t) X] = (uint8_t*) node_map(bytes, X, NH);+ }+ ok = base[(size_t) X] != nullptr;+ if (ok) {+ ram_chunks.push_back(base[(size_t) X]);+ ram_chunk_bytes.push_back(got);+ }+ }+ if (!ok) {+ std::fprintf(stderr, "ds4: no RAM for layer %d's %d copied experts (%.1f GB): they stay in place\n", l.il,+ ns, (double) bytes * NH / 1e9);+ n_copy[(size_t) l.il] = 0;+ continue;+ }+ first[(size_t) l.il] = (int) ram_part.size();+ for (int i = 0; i < ns; ++i)+ ram_part.push_back({base[0] + (size_t) i * part, NH > 1 ? base[1] + (size_t) i * part : nullptr});+ if (spare_of(l)) copy_spare[(size_t) l.il] = first[(size_t) l.il] + ns - 1;+ }+ struct Job {+ int layer, e, slot;+ };+ std::vector<Job> jobs;+ {+ std::vector<int> next = first;+ for (size_t i = 0; i < take; ++i) {+ const int il = hot[i].first;+ if (n_copy[(size_t) il] == 0) continue;+ jobs.push_back({il, hot[i].second, next[(size_t) il]++});+ }+ }+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) { // (file order: the disk reads ahead)+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];+ if (la.mx_file[0] != lb.mx_file[0]) return la.mx_file[0] < lb.mx_file[0];+ return la.mx_off[0] + (uint64_t) a.e * la.mx_stride[0] < lb.mx_off[0] + (uint64_t) b.e * lb.mx_stride[0];+ });+ std::atomic<size_t> ji{0};+ std::atomic<bool> bad{false};+ std::atomic<int64_t> done_b{0};+ std::vector<std::thread> th;+ for (int t = 0; t < 12 && !jobs.empty(); ++t)+ th.emplace_back([&] {+ for (size_t i; (i = ji.fetch_add(1)) < jobs.size() && !bad.load();) {+ const Job& j = jobs[i];+ const Layer& l = L[(size_t) j.layer];+ for (int r = 0; r < 3; ++r) {+ const size_t nb = (size_t) (mx_n(r) / NH) * mx_row_b(l, r);+ for (int X = 0; X < NH; ++X)+ if (!pk.read_at(l.mx_file[r], l.mx_off[r] + (uint64_t) j.e * l.mx_stride[r] + (uint64_t) X * nb, nb,+ ram_part[(size_t) j.slot][(size_t) X] + cp_lay[(size_t) l.il][(size_t) r]))+ bad.store(true);+ }+ const int64_t db = done_b.fetch_add((int64_t) l.blob) + (int64_t) l.blob;+ if (o.verbose && i % 2048 == 0)+ std::fprintf(stderr, "ds4: copying experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(), db / 1e9,+ db / 1e6 / std::max(1.0, now_ms() - t0));+ }+ });+ for (auto& t : th) t.join();+ if (bad.load()) {+ err = "copying the CPU lane's experts from the model's files failed";+ return false;+ }+ for (const Job& j : jobs) L[(size_t) j.layer].ram[(size_t) j.e] = j.slot;+ copy_mode = true;+ ram_held = (int) jobs.size();+ double cb = 0;+ for (const Job& j : jobs) cb += (double) L[(size_t) j.layer].blob;+ std::fprintf(stderr, "ds4: %zu of the CPU lane's %zu experts copied into RAM (%.1f GB, %d NUMA part%s) in %.1f s%s\n",+ jobs.size(), hot.size(), cb / 1e9, NH, NH > 1 ? "s" : "", (now_ms() - t0) / 1000.0,+ jobs.size() < hot.size() ? "; the others read in place" : "");+ // DS4_COPY_PIN: pin the copies, so the prompt path DMAs each expert from where it lies instead of copying it to+ // the pinned staging first (the pass streams 122 GB at 6.8 GB/s through the staging, where the RAM-to-staging+ // and DMA probes are 24 and 28 GB/s). The CPU lane reads the same memory, and pinned 4 KB pages can cost its+ // TLB (an earlier registration measured -15% decode on a 12 GB card, where the CPU lane does nearly all of the+ // decode) - so AUTO (unset) pins when the cards hold at least half the routed experts: the CPU lane is then a+ // minority of the decode and the prompt stream is what hurts. Measured on 4x R9700, 2.25 bpw (share 0.72),+ // three interleaved pairs: the second 8K prompt of a process 12.0/11.9/11.9 s against 28.8/13.8/12.3 s unpinned,+ // decode 35.42/35.50/34.9-36.0 against 35.43/35.45/34.9-36.0 tok/s (no cost). DS4_COPY_PIN=0/1 forces it.+ // DS4_COPY_PIN_RO=1: cudaHostRegisterReadOnly (the GPU only reads it).+ // 0.2.7: DS4_COPY_PIN unset = ON when EITHER condition measured reliable holds: (a) the whole arena is on explicit 2 MB pages (a pin of a+ // THP arena registers 4 KB pages and the DMA rate was erratic on the 12 GB cell: 11.9 s one prompt, 30.4 s the next), or (b) the+ // cards hold at least half the routed experts on RDNA4 (4 x R9700: the pin was reliable in 4/4 runs there, see above). =0 is the off+ // switch, =1 pins any arena.+ const bool pin_copies = strata::ds4::copy_pin_default(env_tri("DS4_COPY_PIN"), arena_all_huge, rdna4(), gpu_expert_share(),+ !ram_chunks.empty(), defaults_026());+ if (pin_copies) {+ const bool ro = std::getenv("DS4_COPY_PIN_RO") != nullptr && std::atoi(std::getenv("DS4_COPY_PIN_RO")) != 0;+ const double tr = now_ms();+ bool ok = true;+ for (size_t i = 0; i < ram_chunks.size() && ok; ++i)+ ok = cudaHostRegister(ram_chunks[i], ram_chunk_bytes[i],+ ro ? cudaHostRegisterReadOnly : cudaHostRegisterPortable) == cudaSuccess;+ if (ok) {+ ram_pinned = true;+ std::fprintf(stderr, "ds4: the copied experts pinned in %.1f s (%s): the prompt path DMAs them in place\n",+ (now_ms() - tr) / 1000.0, ro ? "read-only" : "read-write");+ } else {+ std::fprintf(stderr, "ds4: pinning the copied experts failed: the prompt path stages them\n");+ }+ }+ return true;+ }++ bool load_experts(std::string& err) {+ if (!L.empty() && L[0].lay.fmt == gg::kFmtMXFP4) return load_experts_mx(err);+ // every expert's region, in file order, by a few threads; VRAM ones through a pinned staging blob. In place+ // (file_experts) the CPU lane's experts are not copied: the most-routed of them (the profile's share of their+ // layer) are read once, up to warm_bytes, so the page cache starts with them, and the VRAM ones' pages are+ // dropped from it again+ struct Job {+ int layer, e;+ };+ std::vector<Job> jobs;+ for (const Layer& l : L)+ for (int e = 0; e < l.n_exp; ++e)+ if (!file_experts || l.slot[(size_t) e] >= 0) jobs.push_back({l.il, e});+ uint64_t warm = 0;+ if (file_experts) {+ std::vector<std::pair<double, Job>> cand;+ for (const Layer& l : L) {+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ double sum = 0;+ for (int e = 0; e < l.n_exp; ++e) sum += p[e];+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) cand.push_back({sum > 0 ? p[e] / sum : 0.0, Job{l.il, e}});+ }+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });+ for (const auto& c : cand) {+ const uint64_t b = L[(size_t) c.second.layer].region_bytes[(size_t) c.second.e];+ if (warm + b > warm_bytes) break;+ warm += b;+ jobs.push_back(c.second);+ }+ }+ std::sort(jobs.begin(), jobs.end(), [&](const Job& a, const Job& b) {+ const Layer &la = L[(size_t) a.layer], &lb = L[(size_t) b.layer];+ if (la.region_file[(size_t) a.e] != lb.region_file[(size_t) b.e])+ return la.region_file[(size_t) a.e] < lb.region_file[(size_t) b.e];+ return la.region_off[(size_t) a.e] < lb.region_off[(size_t) b.e];+ });+ std::atomic<size_t> next{0};+ std::atomic<bool> failed{false};+ std::mutex emu;+ std::string ferr;+ const int nth = 12;+ size_t maxr = 0, maxb = 0;+ for (const Layer& l : L) {+ maxb = std::max(maxb, l.blob);+ for (uint64_t r : l.region_bytes) maxr = std::max<size_t>(maxr, r);+ }+ std::atomic<int64_t> done_bytes{0};+ const double t0 = now_ms();+ std::vector<std::thread> th;+ for (int t = 0; t < nth; ++t)+ th.emplace_back([&] {+ std::vector<uint8_t> region(maxr);+ uint8_t *stage = nullptr, *stage2 = nullptr;+ cudaHostAlloc((void**) &stage, maxb, cudaHostAllocPortable);+ cudaHostAlloc((void**) &stage2, maxb, cudaHostAllocPortable);+ int cur_dev = -1;+ for (;;) {+ const size_t i = next.fetch_add(1);+ if (i >= jobs.size() || failed.load()) break;+ const Layer& l = L[(size_t) jobs[i].layer];+ const int e = jobs[i].e;+ if (!pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], l.region_bytes[(size_t) e],+ region.data())) {+ std::lock_guard<std::mutex> lk(emu);+ ferr = "reading layer " + std::to_string(l.il) + " expert " + std::to_string(e) + " failed";+ failed.store(true);+ break;+ }+ if (l.slot[(size_t) e] >= 0) {+ repack(l, region.data(), stage);+ bool up = true;+ for (int m = 0; m < l.nm && up; ++m) { // (a group: each member its part)+ const Layer& x = member(l, m);+ const Dev& d = devs[(size_t) x.dev];+ if (cur_dev != d.id) {+ cudaSetDevice(d.id);+ cur_dev = d.id;+ }+ if (l.nm > 1) blob_to_half(l, stage, m, stage2);+ up = cudaMemcpy(x.vram_base + (size_t) l.slot[(size_t) e] * unit(x), l.nm > 1 ? stage2 : stage,+ unit(x), cudaMemcpyHostToDevice) == cudaSuccess;+ }+ if (!up) {+ std::lock_guard<std::mutex> lk(emu);+ ferr = "uploading layer " + std::to_string(l.il) + " expert " + std::to_string(e);+ failed.store(true);+ break;+ }+ if (file_experts)+ os::advise(pk.fds[(size_t) l.region_file[(size_t) e]], l.region_off[(size_t) e], l.region_bytes[(size_t) e], false);+ } else if (!file_experts) {+ for (int X = 0; X < NH; ++X) repack_part(l, region.data(), X, l.hh[(size_t) e][(size_t) X]);+ }+ const int64_t db = done_bytes.fetch_add((int64_t) l.region_bytes[(size_t) e]) ++ (int64_t) l.region_bytes[(size_t) e];+ if (o.verbose && (i % 1024) == 0)+ std::fprintf(stderr, "ds4: experts %zu / %zu (%.1f GB, %.2f GB/s)\n", i, jobs.size(),+ db / 1e9, db / 1e6 / std::max(1.0, now_ms() - t0));+ }+ if (stage) cudaFreeHost(stage);+ if (stage2) cudaFreeHost(stage2);+ });+ for (auto& t : th) t.join();+ if (failed.load()) {+ err = ferr;+ return false;+ }+ if (file_experts)+ std::fprintf(stderr, "ds4: experts loaded in %.1f s (%.1f GB of the CPU lane's read ahead)\n",+ (now_ms() - t0) / 1000.0, warm / 1e9);+ return true;+ }++ std::string describe() const {+ std::ostringstream s;+ for (size_t di = 0; di < devs.size(); ++di) {+ const Dev& d = devs[di];+ int a = -1, b = -1, ev = 0, et = 0;+ for (const Layer& l : L)+ for (int m = 0; m < l.nm; ++m)+ if (member(l, m).dev == (int) di) {+ if (a < 0) a = l.il;+ b = l.il;+ ev += l.n_vram;+ et += l.n_exp;+ }+ size_t fr = 0, tot = 0;+ cudaSetDevice(d.id);+ cudaMemGetInfo(&fr, &tot);+ const size_t gsz = groups.empty() ? 1 : groups[(size_t) d.grp].mem.size();+ char tp[64] = "";+ if (gsz > 1) std::snprintf(tp, sizeof tp, " (tensor parallel %zu, part %d)", gsz, d.mi);+ char line[384];+ std::snprintf(line, sizeof line,+ "ds4: CUDA%d layers [%d, %d]%s%s: weights %.2f GB, experts %.2f GB (%d of %d in VRAM), caches "+ "%.2f GB, %.0f MiB free\n",+ d.id, a, b, tp, (int) di == head_dev ? " + head" : "", d.weights / 1e9, d.experts / 1e9, ev, et,+ d.caches / 1e9, fr / 1048576.0);+ s << line;+ }+ int nh = 0;+ size_t hb_ = 0;+ for (const Layer& l : L) {+ nh += l.n_exp - l.n_vram;+ hb_ += (size_t) (l.n_exp - l.n_vram) * l.blob;+ }++ if (std::getenv("DS4_PLACE_HASH")) { // debugging: which experts hold a VRAM slot, as one hash (is the placement the same from run to run?)++ uint64_t h = 1469598103934665603ull;++ for (const Layer& l : L)++ for (int e = 0; e < l.n_exp; ++e) { h ^= (uint64_t) (l.slot[(size_t) e] >= 0 ? 1 + l.il * 1000 + e : 0); h *= 1099511628211ull; }++ char hl[96];++ std::snprintf(hl, sizeof hl, "ds4: placement hash %016llx\n", (unsigned long long) h);++ s << hl;++ }+ char line[200];+ if (copy_mode)+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB): %d copied into RAM (%d NUMA "+ "parts), the rest in place in the model's files; %d threads\n",+ nh, hb_ / 1e9, ram_held, NH, cpu_threads());+ else if (file_experts)+ std::snprintf(line, sizeof line,+ "ds4: %d routed experts on the CPU lane (%.1f GB, in place in the model's files), %d threads\n",+ nh, hb_ / 1e9, cpu_threads());+ else+ std::snprintf(line, sizeof line, "ds4: %d routed experts on the CPU lane (%.1f GB of RAM, %d NUMA parts), %d threads\n", nh,+ hb_ / 1e9, NH, cpu_threads());+ s << line;+ return s.str();+ }++ // The most the automatic store may take: all of the trunk's routed experts (the CPU lane's are a part of them; a store larger+ // than they are is never filled), never less than the old 48 GB. A PC whose free RAM holds them all (the 251 GB R9700 box:+ // 115 GB of CPU-lane experts, 1 card, 2.00 set: 12.6 -> 21.4 tokens/s with the whole set resident, misses 15.4 -> 1.5 a token)+ // gets every one; the RAM that the room below leaves (free - 12 GB) is what bounds it elsewhere.+ // Whether the experts the GPUs will not hold (the CPU lane's: the pack's less the GPUs' share) all fit the RAM this process can get beside+ // kRamMargin, so that --mmap-experts auto copies them in (a copied expert is the CPU lane's own memory: no page cache to lose to the+ // Engram tables and the files' churn, no store indirection or miss path). Measured, 1 x R9700 + 251 GB, 2.00 pack, warm: decode +10-15%.+ // --mmap-experts on/off and --ssd-cache N stand as given.+ bool cpu_lane_fits_ram() const {+ if (!pk.gguf || o.mmap_experts >= 0 || o.ssd_cache_gb >= 0.0 || std::getenv("DS4_STORE_GB") || std::getenv("DS4_COPY_GB")) return false;+ if (const char* v = std::getenv("DS4_COPY_ALL")) return std::atoi(v) != 0 && ram_avail() != ~uint64_t{0};+ // An integrated GPU's experts are already in this same RAM (and, above the RAM, in the page file it commits): copying+ // the CPU lane's in beside them takes the pages the GPU's experts live on. Measured on Strix Halo (64 GB, 2.00 set):+ // 82 GB of experts on the GPU, then 53 GB copied in, and an 8K prompt did not finish in 11 minutes (the in-place path is+ // what the 8-hour soak ran). DS4_COPY_ALL=1 still forces it; DS4_COPY_ALL=0 leaves it off everywhere.+ for (const Dev& d : devs)+ if (commit_charged(d)) return false;+ const uint64_t avail = ram_avail();+ if (avail == ~uint64_t{0}) return false;+ double exp_b = 0, room = 0, other_b = 0;+ const double share = gpu_expert_share(&exp_b, &room, &other_b);+ if (exp_b <= 0) return false;+ const double lane = exp_b * (1.0 - std::min(1.0, share));+ return copy_all_fits(lane, avail, (double) kRamMargin, commit_room()); // (Windows: the commit room too, not only the free RAM)+ }+ mutable bool auto_copy_all = false; // (set by store_gb(): the auto size is the CPU lane's experts copied in, no demand store)+ double store_cap_gb() const {+ double b = 0;+ for (const auto& kt : pk.t)+ if (kt.first.find(".ffn.experts.") != std::string::npos && kt.first.rfind("mtp.", 0) != 0) b += (double) kt.second.bytes;+ return std::max(48.0, b / 1e9);+ }+ double store_gb() const {+ static double cached = -2.0; // (decided once: the layout of the blobs depends on it)+ if (cached > -1.5) return cached;+ double gb = o.ssd_cache_gb;+ if (const char* v = std::getenv("DS4_STORE_GB")) gb = std::atof(v);+ if (!pk.gguf) gb = 0.0;+ else if (gb < 0.0 && cpu_lane_fits_ram()) { // auto, and the RAM holds every CPU-lane expert beside kRamMargin: they are copied in at load+ auto_copy_all = true; // (copy_experts), the demand store would only be a second, smaller copy of them+ gb = 0.0;+ } else if (gb < 0.0) { // auto: the RAM free now less 12 GB for the system and the owner's programs, all the experts at most+ const uint64_t avail = ram_avail();+ avail0_ = avail;+ const double room = avail == ~uint64_t{0} ? 0.0 : (double) avail / 1e9 - 12.0;+ gb = room >= 4.0 ? std::min(store_cap_gb(), room) : 0.0;+ }+ return cached = gb;+ }+ // The auto size is decided again here, after the weights are on the GPU: on a unified-memory machine (Strix Halo) the GPU's+ // allocations come out of the same RAM the store would use, and the first estimate (made at the layout, before any upload)+ // saw all of it free. The explicit sizes (--ssd-cache, DS4_STORE_GB) stand.+ mutable uint64_t avail0_ = ~uint64_t{0}; // (free RAM when the first estimate was made, before the weights went to the GPU)+ double store_gb_now() const {+ const double gb = store_gb();+ if (gb <= 0.0 || o.ssd_cache_gb >= 0.0 || std::getenv("DS4_STORE_GB")) return gb;+ const uint64_t avail = ram_avail();+ if (avail == ~uint64_t{0}) return gb;+ // (the prompt buffers and the context's caches are touched after this point and, on such a machine, take RAM too: measured on+ // Strix Halo about 0.11 MB a token of context beyond 16384, a 40960 context ending 2.5 GB lower than a 16384 one)+ const bool uma = avail0_ != ~uint64_t{0} && avail0_ > avail && (double) (avail0_ - avail) / 1e9 >= 4.0;+ const double ctx_gb = uma ? 1.1e-4 * std::max(0.0, (double) o.max_context - 16384.0) : 0.0;+ const double room = (double) avail / 1e9 - 12.0 - ctx_gb;+ const double now = room >= 4.0 ? std::min(store_cap_gb(), room) : 0.0;+ if (now < gb)+ std::fprintf(stderr, "ds4: RAM store: %.1f GB of RAM free after the GPU's weights: store sized %.1f GB, not %.1f\n",+ (double) avail / 1e9, now, gb);+ return std::min(gb, now);+ }+ void init_store() {+ const double gb = store_gb_now();+ if (gb <= 0.0 || L.empty() || L[0].lay.fmt != gg::kFmtMXFP4 || !file_experts) return;+ if (o.ssd_cache_gb < 0.0 && !std::getenv("DS4_STORE_GB")) { // (the auto size: say when it holds less than the CPU lane's experts)+ double exp_b = 0;+ const double share = gpu_expert_share(&exp_b), lane_gb = exp_b * (1.0 - std::min(1.0, share)) / 1e9;+ if (lane_gb > gb + 1.0 && o.verbose)+ std::fprintf(stderr, "ds4: RAM store: the auto size (%.1f GB) holds less than the CPU lane's experts (%.1f GB): the hottest are kept, the "+ "rest is read from the SSD as the prompt picks it (--ssd-cache N sets the size)\n", gb, lane_gb);+ }+ std::vector<ExpertStore::LayerGeo> geo((size_t) g.nl); // (the trunk's layers: the drafter's bigger experts stay in place)+ for (const Layer& l : L) {+ if (l.il >= g.nl) continue;+ ExpertStore::LayerGeo& gl = geo[(size_t) l.il];+ gl.n_exp = l.n_exp;+ for (int r = 0; r < 3; ++r) {+ gl.file[r] = l.mx_file[r];+ gl.off[r] = l.mx_off[r];+ gl.stride[r] = l.mx_stride[r];+ gl.bytes[r] = mx_part_bytes(l, r);+ gl.tr[r] = l.lay.trellis[r];+ }+ gl.blob = l.blob;+ }+ ExpertStore::Cfg cfg;+ cfg.budget_bytes = (size_t) (gb * 1e9);+ cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;+ cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;+ cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);+ cfg.ram_avail = [] { return ram_avail(); };+ cfg.reserve = (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192);+ cfg.vis_percent = std::getenv("DS4_STORE_ZC_FRAC") ? std::atoi(std::getenv("DS4_STORE_ZC_FRAC")) : 50;+ // (the GPU computing the store's experts over PCIe is opt-in, DS4_STORE_ZC=1: measured slower than the CPU lane alone on a+ // 5070 / PCIe 5.0 - the staging copy sits in the layer's critical path - and its answers depend on the timing of which lane+ // took which expert, the CPU lane's are the mmap path's byte for byte)+ const bool want_zc = std::getenv("DS4_STORE_ZC") && std::atoi(std::getenv("DS4_STORE_ZC")) != 0 && devs.size() == 1 &&+ L[0].nm == 1;+ if (want_zc) { // the arena's chunks registered for the GPU to read in place (zero-copy over PCIe)+ const int dev0 = devs[0].id;+ { // (a unified-memory GPU whose budget the weights already fill: the registered store is the same RAM, past the budget the GPU faults)+ cudaDeviceProp zp{};+ size_t zfree = 0, ztotal = 0;+ cudaSetDevice(dev0);+ if (cudaGetDeviceProperties(&zp, dev0) == cudaSuccess && zp.integrated && cudaMemGetInfo(&zfree, &ztotal) == cudaSuccess &&+ (double) zfree < gb * 1e9)+ std::fprintf(stderr,+ "ds4: DS4_STORE_ZC=1 on a GPU that shares the PC's RAM: the weights already fill its memory budget (%.1f GB left, the store would "+ "register up to %.1f GB more), so this will likely fail with a GPU fault. The store is the same RAM either way, zero-copy gains nothing here; "+ "leave DS4_STORE_ZC off.\n", (double) zfree / 1e9, gb);+ }+ cfg.map_chunk = [dev0](void* h, size_t n) -> void* {+ cudaSetDevice(dev0);+ unsigned reg_flags = cudaHostRegisterMapped | cudaHostRegisterPortable;+#if defined(STRATA_USE_HIP)+ // (test arm: DS4_ZC_COARSE=1, hipExtHostRegisterCoarseGrained, which ROCm Core SDK 10.1 honors on Windows; the store is read-only to the GPU)+ if (std::getenv("DS4_ZC_COARSE") && std::atoi(std::getenv("DS4_ZC_COARSE")) != 0) reg_flags |= 0x8u;+#endif+ if (cudaHostRegister(h, n, reg_flags) != cudaSuccess) {+ (void) cudaGetLastError();+ return nullptr;+ }+ void* d = nullptr;+ if (cudaHostGetDevicePointer(&d, h, 0) != cudaSuccess) {+ (void) cudaGetLastError();+ cudaHostUnregister(h);+ return nullptr;+ }+ if (d != h) std::fprintf(stderr, "ds4: RAM store: chunk host %p -> device %p (not the identity)\n", h, d);+ return d;+ };+ cfg.unmap_chunk = [dev0](void* h) {+ cudaSetDevice(dev0);+ cudaHostUnregister(h);+ };+ }+ bool direct = true;+ for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;+ auto st = std::make_unique<ExpertStore>();+ if (!st->init(cfg, geo, pk.dfds, pk.pfds)) {+ std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");+ return;+ }+ store = std::move(st);+ // (the router lookahead's guesses are not read ahead with the store: measured 4.5 against 4.7 tok/s, 62% of the guesses are right and+ // the SSD is the narrow part; DS4_AHEAD=1 turns them on)+ if (std::getenv("DS4_AHEAD") == nullptr) ahead_mode = 0;+ for (Layer& l : L)+ if (l.il < g.nl && want_zc) l.zc.assign((size_t) l.n_exp, 0);+ std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",+ store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,+ (int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");+ if (want_zc) {+ if (!store->gpu_mapped()) {+ std::fprintf(stderr, "ds4: RAM store: the arena could not be mapped for the GPU: zero-copy off\n");+ for (Layer& l : L) l.zc.clear();+ } else {+ zc_stride = (store->max_blob() + 4095) & ~(size_t) 4095;+ cudaSetDevice(devs[0].id);+ if (cudaMalloc((void**) &zc_stage_buf, zc_stride * kPicks) != cudaSuccess) {+ (void) cudaGetLastError();+ zc_stage_buf = nullptr;+ for (Layer& l : L) l.zc.clear();+ } else {+ for (Layer& l : L)+ if (!l.zc.empty() && cudaMalloc((void**) &l.zc_tab, (size_t) l.n_exp * 8) == cudaSuccess) cudaMemset(l.zc_tab, 0, (size_t) l.n_exp * 8);+ else if (!l.zc.empty()) {+ (void) cudaGetLastError();+ l.zc.clear();+ }+ }+ }+ std::fprintf(stderr, "ds4: RAM store: the GPU takes its slots over PCIe (%s), %zu slots kept evictable\n",+ zc_stage_buf != nullptr ? "copied into a staging area at the PCIe rate first" : "off", cfg.reserve);+ }+ }+ // the profile's hottest experts (VRAM's left out) read into the store before the first token (DS4_STORE_PRELOAD=0: not)+ void preload_store() {+ if (!store) return;+ if (std::getenv("DS4_STORE_PRELOAD") && std::atoi(std::getenv("DS4_STORE_PRELOAD")) == 0) return;+ std::vector<std::pair<double, std::pair<int, int>>> cand;+ double tot = 0;+ for (const Layer& l : L) {+ if (l.il >= g.nl) continue;+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ for (int e = 0; e < l.n_exp; ++e) tot += p[e];+ }+ if (tot <= 0) return; // (no profile yet: the first tokens fill it)+ for (const Layer& l : L) {+ if (l.il >= g.nl) continue;+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0 && p[e] > 0) cand.push_back({(double) p[e] / (double) l.blob, {l.il, e}});+ }+ std::stable_sort(cand.begin(), cand.end(), [](const auto& a, const auto& b) { return a.first > b.first; });+ std::vector<std::pair<int, int>> list;+ list.reserve(cand.size());+ for (const auto& c : cand) list.push_back(c.second);+ // The slots the CPU lane's experts leave over: the experts the VRAM cache holds that the profile counts least (the cache gives+ // up its coldest to the CPU lane, and a store without them reads each from the SSD while the CPU lane waits - 1.8 misses a+ // token on the 251 GB R9700 box, 2.9 ms of a 49 ms token), then the lane's experts the profile never saw. Last in the list: they+ // take only the slots the likely ones left. DS4_STORE_SPARE=0: off.+ if (strata::ds4::store_feature_default(env_tri("DS4_STORE_SPARE"), defaults_026())) {+ std::vector<std::pair<double, std::pair<int, int>>> vr, zero;+ for (const Layer& l : L) {+ if (l.il >= g.nl) continue;+ const uint32_t* p = prof.data() + (size_t) l.il * g.n_exp;+ for (int e = 0; e < l.n_exp; ++e) {+ if (l.slot[(size_t) e] >= 0) vr.push_back({(double) p[e] / (double) l.blob, {l.il, e}});+ else if (p[e] == 0) zero.push_back({0.0, {l.il, e}});+ }+ }+ std::stable_sort(vr.begin(), vr.end(), [](const auto& a, const auto& b) { return a.first < b.first; });+ for (const auto& c : vr) list.push_back(c.second);+ for (const auto& c : zero) list.push_back(c.second);+ }+ const double t0 = now_ms();+ // (with the spare experts in the list the reserve is theirs: last in the list, so the oldest ticks, so the first a demand read replaces)+ const bool spare_on = strata::ds4::store_feature_default(env_tri("DS4_STORE_SPARE"), defaults_026());+ const size_t leave = spare_on ? 0 : (size_t) (std::getenv("DS4_STORE_RESERVE") ? std::atoi(std::getenv("DS4_STORE_RESERVE")) : 192) + 64;+ // The first 48 GB (what a store held before it could take every expert) are read before the first token; the rest of the+ // hottest-first list is read after it, in pieces of 128 experts on a thread of its own, behind the model's own reads (a+ // guess's priority) - the same bytes. Opt-in (DS4_STORE_BG=1): "ready" comes 9 s sooner on the 251 GB R9700 box (24 s against 34), but a prompt sent at+ // once then waits for the same SSD (200 tokens: 18.7 s against 9.0), so the first answer starts no sooner and decode after+ // it measured 3.7% lower; it only pays when the first prompt comes more than ~10 s after ready.+ const size_t first = !(std::getenv("DS4_STORE_BG") && std::atoi(std::getenv("DS4_STORE_BG")) == 1)+ ? list.size()+ : std::min(list.size(), (size_t) (48e9 / (double) std::max<size_t>(1, L[0].blob)));+ const std::vector<std::pair<int, int>> head(list.begin(), list.begin() + (long) first), tail(list.begin() + (long) first, list.end());+ const size_t n = store->preload(head, leave);+ std::fprintf(stderr, "ds4: RAM store: %zu of the profile's hottest experts read in (%.1f GB) in %.1f s%s\n", n,+ (double) n * (double) L[0].blob / 1e9, (now_ms() - t0) / 1000.0,+ tail.empty() ? "" : "; the rest is read in the background");+ if (tail.empty()) return;+ store_bg = std::thread([this, tail, leave] {+ const double tb = now_ms();+ size_t got = 0;+ std::vector<std::pair<int, int>> piece;+ for (size_t i = 0; i < tail.size() && !quit.load(std::memory_order_relaxed); i += 128) {+ piece.assign(tail.begin() + (long) i, tail.begin() + (long) std::min(tail.size(), i + 128));+ got += store->preload(piece, leave);+ }+ std::fprintf(stderr, "ds4: RAM store: %zu more experts (%.1f GB) read in the background in %.1f s\n", got,+ (double) got * (double) L[0].blob / 1e9, (now_ms() - tb) / 1000.0);+ });+ }+ // at a window boundary: the store's ready slots become the device's (zero-copy) experts, taken back when it needs the room;+ // each touched layer's whole table is written again (VRAM slots, the store's, 0), in stream order before the next window+ void store_sync() {+ if (!store || !store->gpu_mapped()) return;+ std::vector<ExpertStore::Vis> ev;+ store->sync(ev, [&](int layer, int e) { return L[(size_t) layer].slot[(size_t) e] < 0; });+ if (ev.empty()) return;+ std::vector<char> dirty(L.size(), 0);+ for (const auto& v : ev) {+ Layer& l = L[(size_t) v.layer];+ if (l.zc.empty()) continue;+ l.zc[(size_t) v.e] = v.dev != 0 ? 1 : 0;+ dirty[(size_t) v.layer] = 1;+ }+ for (Layer& l : L) {+ if (!dirty[(size_t) l.il]) continue;+ if (l.zc_tab == nullptr) continue;+ std::vector<unsigned long long> tab((size_t) l.n_exp, 0ull);+ for (int e = 0; e < l.n_exp; ++e)+ if (l.zc[(size_t) e]) {+ const int s = l.slot[(size_t) e] < 0 ? store->find(l.il, e) : -1;+ tab[(size_t) e] = s >= 0 ? store->dev_blob(s) : 0ull;+ if (s < 0) l.zc[(size_t) e] = 0;+ }+ cudaSetDevice(devs[0].id);+ cudaMemcpyAsync(l.zc_tab, tab.data(), tab.size() * 8, cudaMemcpyHostToDevice, devs[0].st);+ }+ }++ int cpu_threads() const {+ int n = 0;+ for (const auto& pl : pools) n += pl->size() - 1;+ return n + 1;+ }+ // the CPU lane's workers: a pool per NUMA node (NH of them) on its cores; in place with one part, one pool over+ // every node's cores (the pages lie where the cache put them)+ void make_pools() {+ std::vector<std::vector<int>> cs = cores;+ if ((int) cs.size() < NH && !cs.empty()) { // (DS4_COPY_NUMA=2 on one node, a test: its cores split in NH)+ std::vector<int> all = cs[0];+ cs.assign((size_t) NH, {});+ for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);+ }+ static const int lane_free = [] {+ const char* v = std::getenv("STRATA_DS4_LANE_FREE");+ return v != nullptr ? std::max(0, std::atoi(v)) : 2;+ }();+ size_t lane_cpus = 0;+ for (const auto& v : cs) lane_cpus += v.size();+ const bool lane_takes_all = lane_cpus >= (size_t) std::max(1u, std::thread::hardware_concurrency());+ for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {+ std::vector<int> c = cs[(size_t) X];+ if (file_experts && NH == 1)+ for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());+ if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)+ // cores of the first pool left free beside the host thread's, for the thread that launches the decode's kernels+ // (and waits on the device) and whatever else the PC runs: with a worker spinning on every other logical CPU+ // that thread waits for a core (Project Maya #359, a 20-core Windows 11 PC: decode 28.3 -> 55.5 tok/s). Only+ // when the lane would take every logical CPU and --cpu-threads is not given.+ // STRATA_DS4_LANE_FREE: the cores left free (2 by default, 0: as before)+ if (X == 0 && o.cpu_threads <= 0 && lane_free > 0 && (int) c.size() > lane_free + 1 && lane_takes_all)+ c.resize(c.size() - (size_t) lane_free);+ int nw = (int) c.size();+ if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));+ pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)+ }+ (void) gg::mx_tier(); // (measured here, not on the first token)+ if (o.verbose) {+ int nthr = 0;+ for (auto& p : pools) nthr += p->size();+ std::fprintf(stderr, "ds4: CPU lane: %d threads, MXFP4 rows %s (%s)\n", nthr, gg::mx_tier_name(gg::mx_tier()),+ gg::mx_tier_report());+ }+ }++ void reset() {+ pos = 0;+ toks.clear();+ ctoks.clear();+ slot_held = false;+ for (Check& c0 : ck) c0.pos = -1;+ for (Layer& l0 : L)+ for (int m = 0; m < l0.nm; ++m) {+ Layer& l = member(l0, m);+ Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaMemsetAsync(l.win, 0, (size_t) g.win * win_rb(), d.st);+ if (l.c_kv) cudaMemsetAsync(l.c_kv, 0, (size_t) l.ratio * g.hd * 4, d.st);+ if (l.c_score) cudaMemsetAsync(l.c_score, 0, (size_t) l.ratio * g.hd * 4, d.st);+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ }++ // the state at pos into checkpoint `slot` (the window rings and the compressors' partial groups: the rest of the+ // state is indexed by position, and what lies past a restored position is written again)+ void checkpoint(int slot) {+ Check& c = ck[slot & (kChecks - 1)];+ c.pos = pos;+ c.win.resize(L.size());+ c.comp.resize(L.size() * 2);+ for (size_t i = 0; i < L.size(); ++i) {+ Layer& l = L[i];+ Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ c.win[i].resize((size_t) g.win * win_rb());+ cudaMemcpyAsync(c.win[i].data(), l.win, c.win[i].size(), cudaMemcpyDeviceToHost, d.st);+ if (l.c_kv) {+ c.comp[2 * i].resize((size_t) l.ratio * g.hd);+ c.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);+ cudaMemcpyAsync(c.comp[2 * i].data(), l.c_kv, c.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, d.st);+ cudaMemcpyAsync(c.comp[2 * i + 1].data(), l.c_score, c.comp[2 * i + 1].size() * 4,+ cudaMemcpyDeviceToHost, d.st);+ }+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ }++ // the checkpoint at or before position limit (the state's tokens agree with a new prompt up to there), or -1+ int best_check(int64_t limit) const {+ int b = -1;+ for (int i = 0; i < kChecks; ++i)+ if (ck[i].pos >= 0 && ck[i].pos <= limit && ck[i].pos <= pos && (b < 0 || ck[i].pos > ck[b].pos)) b = i;+ return b;+ }+ int64_t restorable(int64_t limit) const {+ const int b = best_check(limit);+ return b < 0 ? -1 : ck[b].pos;+ }++ // checkpoint 1 (the position before the last prompt's end) becomes the anchor too+ void pin() {+ if (ck[1].pos > 0) ck[2] = ck[1];+ }+ // the checkpoints the state can go back to (what a slot of it keeps), latest first+ std::vector<int64_t> check_positions() const {+ std::vector<int64_t> v;+ for (const Check& c : ck)+ if (c.pos > 0 && c.pos <= pos && c.win.size() == L.size() &&+ std::find(v.begin(), v.end(), c.pos) == v.end())+ v.push_back(c.pos);+ std::sort(v.rbegin(), v.rend());+ return v;+ }+ // the checkpoints a slot keeps (each with its role: 0, 1 or the anchor)+ std::vector<int> slot_checks() const {+ std::vector<int> v;+ for (int i = 0; i < kChecks; ++i)+ if (ck[i].pos > 0 && ck[i].pos <= pos && ck[i].win.size() == L.size()) v.push_back(i);+ return v;+ }+ size_t check_bytes() const {+ size_t n = 16;+ for (const Layer& l : L) n += (size_t) g.win * win_rb() + (l.c_kv ? 2 * (size_t) l.ratio * g.hd * 4 : 0);+ return n;+ }++ bool restore(int64_t limit) {+ const int b = best_check(limit);+ if (b < 0) return false;+ const Check& c = ck[b];+ for (size_t i = 0; i < L.size(); ++i)+ for (int m = 0; m < L[i].nm; ++m) { // (a group's members hold the same state: the lead's copy)+ Layer& l = member(L[i], m);+ Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaMemcpyAsync(l.win, c.win[i].data(), c.win[i].size(), cudaMemcpyHostToDevice, d.st);+ if (l.c_kv) {+ cudaMemcpyAsync(l.c_kv, c.comp[2 * i].data(), c.comp[2 * i].size() * 4, cudaMemcpyHostToDevice, d.st);+ cudaMemcpyAsync(l.c_score, c.comp[2 * i + 1].data(), c.comp[2 * i + 1].size() * 4,+ cudaMemcpyHostToDevice, d.st);+ }+ }+ for (Dev& d : devs) {+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ pos = c.pos;+ toks.resize((size_t) pos);+ ctoks.resize((size_t) pos);+ slot_held = false;+ for (Check& o2 : ck) // a later checkpoint saw tokens the state no longer has+ if (o2.pos > pos) o2.pos = -1;+ return true;+ }++ // ---- the kept conversations (a slot): the state out of the model and back, through a file or a host buffer.+ // A prompt that continues the state is reused in place (checkpoint / restore, the compressed KV left where it is);+ // a conversation that leaves the model for another one is a slot - the whole state: the position, its tokens,+ // every window ring (the DSpark blocks' included), the compressors' partial groups and each KV source's compressed+ // KV and index keys up to it (rows 0 .. pos / ratio, the groups its position completed; one the prompt has not+ // completed yet lives in the compressor's partial group). A tensor parallel group's members hold the same state,+ // so one member's ring is saved and written back into all of them. Text only: a conversation that holds pictures+ // is not a slot (the Engram history's rows are the tokens' again, no -1 among them).+ static constexpr uint64_t kSlotMagic = 0x31544f4c53345344ull; // "DS4SLOT1" (little endian)++ int64_t src_rows(const Src& s) const { return pos / std::max(1, s.ratio); }++ // the slot's bytes: the header, the tokens, every layer's ring and partial group, every source's rows, the end+ size_t slot_bytes() const {+ size_t n = 8 + 8 + slot_identity().size() + 8 + (size_t) pos * 4;+ for (const Layer& l : L) n += 8 + (size_t) g.win * win_rb() + 8 + (l.c_kv ? 2 * (size_t) l.ratio * g.hd * 4 : 0);+ for (const Src& s : srcs) n += 8 + (size_t) src_rows(s) * (ckv_rb() + ik_rb());+ n += 8 + slot_checks().size() * check_bytes(); // (the checkpoints: what the slot can be taken back at)+ return n + 8;+ }++ uint64_t slot_save(std::vector<uint8_t>& out, std::string& err) {+ if (slot_held) {+ err = "it is already set aside";+ return 0;+ }+ if (pos <= 0) {+ err = "there is no conversation";+ return 0;+ }+ if ((int64_t) toks.size() != pos) {+ err = "the state's tokens do not match its position";+ return 0;+ }+ if (!imgs.empty()) {+ err = "it holds pictures";+ return 0;+ }+ const std::string id = slot_identity();+ out.assign(slot_bytes(), 0);+ uint8_t* p = out.data();+ size_t off = 0;+ const auto put = [&](const void* s, size_t b) {+ std::memcpy(p + off, s, b);+ off += b;+ };+ const uint64_t il = id.size(), n = (uint64_t) pos;+ put(&kSlotMagic, 8);+ put(&il, 8);+ put(id.data(), (size_t) il);+ put(&n, 8);+ put(toks.data(), (size_t) n * 4);+ for (Dev& d : devs) { // (the devices' work ends before their state is read)+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ for (size_t i = 0; i < L.size(); ++i) {+ const Layer& l = L[i];+ const Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ const uint64_t wb = (uint64_t) g.win * win_rb();+ put(&wb, 8);+ if (cudaMemcpy(p + off, l.win, (size_t) wb, cudaMemcpyDeviceToHost) != cudaSuccess) {+ err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());+ return 0;+ }+ off += (size_t) wb;+ const uint64_t cn = l.c_kv ? (uint64_t) l.ratio * (uint64_t) g.hd : 0;+ put(&cn, 8);+ if (cn > 0) {+ const size_t cb = (size_t) cn * 4;+ if (cudaMemcpy(p + off, l.c_kv, cb, cudaMemcpyDeviceToHost) != cudaSuccess ||+ cudaMemcpy(p + off + cb, l.c_score, cb, cudaMemcpyDeviceToHost) != cudaSuccess) {+ err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());+ return 0;+ }+ off += 2 * cb;+ }+ }+ for (const Src& s : srcs) {+ const uint64_t rows = (uint64_t) src_rows(s);+ put(&rows, 8);+ if (rows == 0) continue;+ const int di = L[(size_t) s.layer].dev;+ const uint8_t* ckv = s.ckv[(size_t) di];+ const uint8_t* ik = s.ik[(size_t) di]; // (a pack whose sources are not index sources: none, zeros)+ if (ckv == nullptr) {+ err = "source layer " + std::to_string(s.layer) + "'s rows are not on its device";+ return 0;+ }+ const size_t cb = (size_t) rows * ckv_rb(), ib = (size_t) rows * ik_rb();+ cudaSetDevice(devs[(size_t) di].id);+ if (cudaMemcpy(p + off, ckv, cb, cudaMemcpyDeviceToHost) != cudaSuccess ||+ (ik != nullptr && cudaMemcpy(p + off + cb, ik, ib, cudaMemcpyDeviceToHost) != cudaSuccess)) {+ err = "source layer " + std::to_string(s.layer) + "'s compressed KV: " + cudaGetErrorString(cudaGetLastError());+ return 0;+ }+ off += cb + ib;+ }+ {+ const std::vector<int> cs = slot_checks();+ const uint64_t nc = cs.size();+ put(&nc, 8);+ for (int k : cs) {+ const Check& c = ck[k];+ const uint64_t role = (uint64_t) k, cp = (uint64_t) c.pos;+ put(&role, 8);+ put(&cp, 8);+ for (size_t i = 0; i < L.size(); ++i) {+ put(c.win[i].data(), (size_t) g.win * win_rb());+ if (L[i].c_kv) {+ put(c.comp[2 * i].data(), (size_t) L[i].ratio * g.hd * 4);+ put(c.comp[2 * i + 1].data(), (size_t) L[i].ratio * g.hd * 4);+ }+ }+ }+ }+ if (off + 8 != out.size()) {+ err = "the state's size does not add up";+ return 0;+ }+ put(&kSlotMagic, 8);+ slot_held = true;+ return (uint64_t) off;+ }++ bool slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err) {+ size_t off = 0;+ bool wrote = false, ok = false;+ // a load that stops after some of another conversation's state went into the layers leaves a state no+ // checkpoint can repair: it is reset (the prompt is read from token 0), never served from the mix+ struct FailClosed {+ Impl& p;+ const bool& wrote;+ const bool& ok;+ ~FailClosed() {+ if (!ok && wrote) p.reset();+ }+ } guard{*this, wrote, ok};+ const auto room = [&](size_t b) { return b <= bytes && off <= bytes - b; };+ uint64_t magic = 0, il = 0, n = 0;+ if (!room(16)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&magic, data + off, 8);+ std::memcpy(&il, data + off + 8, 8);+ off += 16;+ if (magic != kSlotMagic || il == 0 || il > (1u << 20)) {+ err = "not a saved conversation";+ return false;+ }+ if (!room((size_t) il + 8)) {+ err = "the slot is cut short";+ return false;+ }+ {+ std::string id((size_t) il, '\0');+ std::memcpy(id.data(), data + off, (size_t) il);+ off += (size_t) il;+ std::memcpy(&n, data + off, 8);+ off += 8;+ if (id != slot_identity()) {+ err = "another model's, or another engine version's, conversation";+ return false;+ }+ }+ if (n != (uint64_t) n_pos || n_pos <= 0 || n_pos > o.max_context || !room((size_t) n * 4)) {+ err = "the conversation's position is not this prompt's";+ return false;+ }+ std::vector<int32_t> toks_in((size_t) n);+ std::memcpy(toks_in.data(), data + off, (size_t) n * 4);+ off += (size_t) n * 4;+ for (Dev& d : devs) { // (their work ends before the state is written over)+ cudaSetDevice(d.id);+ cudaStreamSynchronize(d.st);+ }+ for (size_t i = 0; i < L.size(); ++i) {+ Layer& l = L[i];+ const Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ uint64_t wb = 0, cn = 0;+ if (!room(8)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&wb, data + off, 8);+ off += 8;+ if (wb != (uint64_t) g.win * win_rb() || !room((size_t) wb + 8)) {+ err = l.name + "window ring of another size";+ return false;+ }+ wrote = true;+ for (int m = 0; m < l.nm; ++m) {+ Layer& lm = member(l, m);+ if (cudaMemcpy(lm.win, data + off, (size_t) wb, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = l.name + "window ring: " + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ }+ off += (size_t) wb;+ std::memcpy(&cn, data + off, 8);+ off += 8;+ if (cn != (l.c_kv ? (uint64_t) l.ratio * (uint64_t) g.hd : 0)) {+ err = l.name + "compressor's group of another size";+ return false;+ }+ if (cn > 0) {+ const size_t cb = (size_t) cn * 4;+ if (!room(2 * cb)) {+ err = "the slot is cut short";+ return false;+ }+ for (int m = 0; m < l.nm; ++m) {+ Layer& lm = member(l, m);+ if (cudaMemcpy(lm.c_kv, data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess ||+ cudaMemcpy(lm.c_score, data + off + cb, cb, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = l.name + "compressor's group: " + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ }+ off += 2 * cb;+ }+ }+ for (const Src& s : srcs) {+ uint64_t rows = 0;+ if (!room(8)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&rows, data + off, 8);+ off += 8;+ if (rows != (uint64_t) (n_pos / std::max(1, s.ratio))) {+ err = "source layer " + std::to_string(s.layer) + "'s rows are not this conversation's";+ return false;+ }+ if (rows == 0) continue;+ const size_t cb = (size_t) rows * ckv_rb(), ib = (size_t) rows * ik_rb();+ if (!room(cb + ib)) {+ err = "the slot is cut short";+ return false;+ }+ for (size_t di = 0; di < devs.size(); ++di) {+ if (s.ckv[di] == nullptr && s.ik[di] == nullptr) continue;+ cudaSetDevice(devs[di].id);+ if (s.ckv[di] != nullptr && cudaMemcpy(s.ckv[di], data + off, cb, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = "source layer " + std::to_string(s.layer) + "'s compressed KV: " + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ if (s.ik[di] != nullptr &&+ cudaMemcpy(s.ik[di], data + off + cb, ib, cudaMemcpyHostToDevice) != cudaSuccess) {+ err = "source layer " + std::to_string(s.layer) + "'s index keys: " + cudaGetErrorString(cudaGetLastError());+ return false;+ }+ }+ off += cb + ib;+ }+ Check got[kChecks]; // the slot's checkpoints: the positions it can be taken back at+ {+ uint64_t nc = 0;+ if (!room(8)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&nc, data + off, 8);+ off += 8;+ if (nc > (uint64_t) kChecks) {+ err = "the slot's checkpoints are damaged";+ return false;+ }+ for (uint64_t k = 0; k < nc; ++k) {+ uint64_t role = 0, cp = 0;+ if (!room(16)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&role, data + off, 8);+ std::memcpy(&cp, data + off + 8, 8);+ off += 16;+ if (role >= (uint64_t) kChecks || got[role].pos >= 0 || cp == 0 || cp > (uint64_t) n_pos) {+ err = "the slot's checkpoints are damaged";+ return false;+ }+ Check& c = got[role];+ c.pos = (int64_t) cp;+ c.win.resize(L.size());+ c.comp.resize(L.size() * 2);+ for (size_t i = 0; i < L.size(); ++i) {+ const size_t wb = (size_t) g.win * win_rb(), cb = L[i].c_kv ? (size_t) L[i].ratio * g.hd * 4 : 0;+ if (!room(wb + 2 * cb)) {+ err = "the slot is cut short";+ return false;+ }+ c.win[i].assign(data + off, data + off + wb);+ off += wb;+ if (cb > 0) {+ c.comp[2 * i].resize(cb / 4);+ c.comp[2 * i + 1].resize(cb / 4);+ std::memcpy(c.comp[2 * i].data(), data + off, cb);+ std::memcpy(c.comp[2 * i + 1].data(), data + off + cb, cb);+ off += 2 * cb;+ }+ }+ }+ }+ uint64_t end = 0;+ if (!room(8)) {+ err = "the slot is cut short";+ return false;+ }+ std::memcpy(&end, data + off, 8);+ off += 8;+ if (end != kSlotMagic) {+ err = "the slot does not end where it should";+ return false;+ }+ pos = n_pos;+ toks.swap(toks_in);+ ctoks.resize((size_t) pos);+ for (size_t i = 0; i < (size_t) pos; ++i)+ ctoks[i] = toks[i] < pk.eg_vocab ? pk.eg_map[(size_t) toks[i]] : 0;+ for (int k = 0; k < kChecks; ++k) ck[k] = std::move(got[k]); // (the slot's own; the old ones held the state+ // this one replaced)+ slot_held = false;+ ok = true;+ return true;+ }++ std::string slot_identity() const {+ const auto size_of = [](const std::string& p) {+ std::error_code ec;+ const auto s = std::filesystem::file_size(p, ec);+ return ec ? 0ull : (unsigned long long) s;+ };+ std::ostringstream id;+ id << "strata-ds4-session-3 engine=" << STRATA_VERSION << " mtp=" << g.n_mtp << " dim=" << g.dim << " hd=" << g.hd+ << " win=" << g.win << " ratios=";+ for (int r : g.ratios) id << r << ",";+ id << " kv=";+ for (int v : g.kv_src) id << v << ",";+ id << " ix=";+ for (int v : g.ix_src) id << v << ",";+ id << " ds4.txt:" << size_of(pk.dir + "/ds4.txt") << " engram.bin:" << size_of(pk.dir + "/engram.bin")+ << " tensors.txt:" << size_of(pk.dir + "/tensors.txt");+ for (size_t i = 0; i < pk.files.size(); ++i)+ id << " " << pk.files[i] << ":" << os::file_size(pk.fds[i]);+ return id.str();+ }++ size_t bounce_bytes() const {+ return (size_t) kWin * ((size_t) g.hc * g.dim * 4 + 8 * 4 + topk_stride() * 4 + cand_stride() * 4);+ }+ size_t topk_stride() const { return (size_t) std::max(g.ix_topk, 32); }+ size_t cand_stride() const { return (size_t) std::max(g.cand_topk, 32); }+ size_t bm_stride() const { return (size_t) (o.max_context + 64) / std::max(1, g.cand_blk) + 64; }+ static constexpr int kSlots = kWin * kPicks; // the CPU lane's (token, expert) rows a window can need++ // ---- the vision tower (the checkpoint's vision.* / aligner.*) on the first GPU, its weights streamed from the+ // host; a prompt's pictures as their spans' rows (set_images)+ VisionTower vt;+ VisionGeo vg;+ int img_tok = -1;+ std::string vision_err; // why the tower did not load (pictures off), or empty+ std::vector<uint16_t> img_mark[3]; // image_start, image_newline, image_end: BF16 [dim]+ std::vector<Image> imgs; // the pictures of the prompt being fed, by start+ // a BF16 (or, rounded to nearest even, F32) tensor of n values into dst+ bool read_bf16(const Tensor& t, size_t n, void* dst) {+ if (t.dtype == "BF16") return t.bytes == n * 2 && pk.read(t, dst);+ if (t.dtype != "F32" || t.bytes != n * 4) return false;+ std::vector<float> f(n);+ if (!pk.read(t, f.data())) return false;+ uint16_t* o = (uint16_t*) dst;+ for (size_t i = 0; i < n; ++i) {+ uint32_t u;+ std::memcpy(&u, &f[i], 4);+ o[i] = (uint16_t) ((u + 0x7FFFu + ((u >> 16) & 1u)) >> 16);+ }+ return true;+ }+ bool load_vision(std::string& err) {+ vg.layers = (int) pk.num("vision.num_hidden_layers", 0);+ const int tok = (int) pk.num("image_token_id", -1);+ if (vg.layers <= 0 || tok < 0 || pk.get("vision.patch_embed.proj.weight") == nullptr) return true; // none+ vg.dim = (int) pk.num("vision.hidden_size", 1024), vg.heads = (int) pk.num("vision.num_attention_heads", 16);+ vg.inter = (int) pk.num("vision.intermediate_size", 2816), vg.patch = (int) pk.num("vision.patch_size", 14);+ vg.merge = (int) pk.num("vision.downsample_ratio", 3), vg.theta = (float) pk.num("vision.rope_theta", 1e4);+ vg.max_tokens = (int) pk.num("vision.max_image_tokens", 1024), vg.out = g.dim;+ const char* mark[3] = {"image_start", "image_newline", "image_end"};+ for (int i = 0; i < 3; ++i) {+ const Tensor* t = pk.get(mark[i]);+ img_mark[i].resize((size_t) g.dim);+ if (t == nullptr || !read_bf16(*t, (size_t) g.dim, img_mark[i].data())) {+ err = std::string(mark[i]) + ": not a BF16 (or F32) row of the model's width";+ return false;+ }+ }+ const double t0 = now_ms();+ Dev& d = devs[0];+ // the tower's tensors as BF16: the checkpoint's own, or a GGUF's mmproj (F32 norms / biases / marker rows, which+ // came from BF16, converted back exactly; mlp.w1 as the two files' gate and up, the halves of its rows)+ auto read = [this](const std::string& n, std::vector<int64_t>& shape, void* dst) {+ const Tensor* t = pk.get(n);+ if (t == nullptr && n.size() > 13 && n.compare(n.size() - 13, 13, "mlp.w1.weight") == 0) {+ const std::string pre = n.substr(0, n.size() - 13);+ const Tensor* tg = pk.get(pre + "mlp.gate.weight");+ const Tensor* tu = pk.get(pre + "mlp.up.weight");+ if (tg == nullptr || tu == nullptr || tg->shape.size() != 2 || tg->shape != tu->shape) return false;+ shape = {tg->shape[0] * 2, tg->shape[1]};+ const size_t half = (size_t) tg->shape[0] * (size_t) tg->shape[1];+ return dst == nullptr || (read_bf16(*tg, half, dst) && read_bf16(*tu, half, (uint16_t*) dst + half));+ }+ if (t == nullptr) return false;+ shape = t->shape;+ size_t n_el = 1;+ for (int64_t x : t->shape) n_el *= (size_t) x;+ return dst == nullptr || read_bf16(*t, n_el, dst);+ };+ if (!vt.load(vg, d.id, read, err)) return false;+ img_tok = tok;+ CK(cudaSetDevice(d.id));+ CK(cudaMemGetInfo(&d.free0, &d.total)); // (the split sees what the tower left)+ if (o.verbose)+ std::fprintf(stderr, "ds4: vision tower (%d blocks) on CUDA%d: %.0f MiB, up to %d patches, %.1f s\n",+ vg.layers, d.id, (double) vt.device_bytes() / (1 << 20), vt.max_patches(),+ (now_ms() - t0) / 1000.0);+ return true;+ }+ bool encode_image(const uint8_t* rgb, int w, int h, std::vector<uint16_t>& span, std::string& err) {+ if (img_tok < 0) {+ err = "this model has no vision tower (or it is off)";+ return false;+ }+ std::vector<uint16_t> rows;+ int nh = 0, nw = 0;+ if (!vt.encode(rgb, w, h, rows, nh, nw, err)) return false;+ const size_t D = (size_t) g.dim;+ span.resize(((size_t) nh * (nw + 1) + 2) * D);+ uint16_t* o = span.data();+ auto put = [&](const uint16_t* r) {+ std::memcpy(o, r, D * 2);+ o += D;+ };+ put(img_mark[0].data());+ for (int i = 0; i < nh; ++i) {+ for (int j = 0; j < nw; ++j) put(rows.data() + ((size_t) i * nw + j) * D);+ put(img_mark[1].data());+ }+ put(img_mark[2].data());+ return true;+ }+ // the row of a picture's span at position p (null: p is in none)+ const uint16_t* image_row(int64_t p) const {+ if (imgs.empty()) return nullptr;+ auto it = std::upper_bound(imgs.begin(), imgs.end(), p, [](int64_t v, const Image& im) { return v < im.start; });+ if (it == imgs.begin()) return nullptr;+ const Image& im = *(it - 1);+ const int64_t r = p - im.start;+ if (!im.rows || r >= (int64_t) (im.rows->size() / (size_t) g.dim)) return nullptr;+ return im.rows->data() + (size_t) r * g.dim;+ }+ // token tk's embedding row at position p: inside a picture's span, the picture's+ const uint16_t* emb_row(int64_t p, int tk) const {+ const uint16_t* r = tk == img_tok ? image_row(p) : nullptr;+ return r != nullptr ? r : embed_pinned + (size_t) tk * g.dim;+ }+ bool in_image(int64_t p) const { return img_tok >= 0 && toks[(size_t) p] == img_tok && image_row(p) != nullptr; }+ // the Engram's compressed id of token tk at p (-1: a picture's, which no n-gram spans)+ int32_t ctok(int64_t p, int tk) const {+ if (tk == img_tok && image_row(p) != nullptr) return -1;+ return tk < pk.eg_vocab ? pk.eg_map[(size_t) tk] : 0;+ }+ // a chunk's image rows (route()'s bits); false: none+ bool vl_bits(int64_t p0, int T, VlBits& b) const {+ if (imgs.empty()) return false;+ std::memset(&b, 0, sizeof b);+ bool any = false;+ for (int t = 0; t < T && t < kVlRows; ++t)+ if (in_image(p0 + t)) {+ b.w[t >> 5] |= 1u << (t & 31);+ any = true;+ }+ return any;+ }++ // ---- Engram: a window's rows of one table (Engram layer li), tokens p0 .. p0 + T - 1 into dst ([token] of [cols]+ // [hd] codes then the scales; nullptr: h_eg). eg_issue serves the rows already fetched from the row cache (and a+ // row the window needs twice from one read) and hands the rest to the reader's threads; eg_finish waits for them,+ // copies the repeats and keeps the new rows in the cache. The rows are the file's bytes either way.+ void eg_issue(int64_t p0, int T, int li, uint8_t* dst, EgBatch& b) {+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, hd = pk.eg_hd;+ const size_t per = (size_t) cols * (hd + hd / 32);+ b.clear();+ b.reads.reserve((size_t) T * cols);+ const std::string P = "layers." + std::to_string(g.eg_layers[(size_t) li]) + ".engram.embed.";+ const Tensor* tw = pk.get(P + "weight");+ const Tensor* ts = pk.get(P + "scale");+ const int fmt = eg_fmt[(size_t) li];+ const bool mx8 = fmt == 0; // (a GGUF: the scales between the codes, one per 32)+ const bool q6k = fmt == 1; // (a GGUF: one Q6_K superblock a 256 values, its scales inside)+ const bool iq4 = fmt == 2; // (a GGUF: 18-byte blocks a 32 values, d and 16 nibble bytes, no scales)+ std::unordered_map<uint64_t, int> first; // a row this batch reads already -> its read+ int64_t hits = 0;+ {+ std::lock_guard<std::mutex> lk(eg_cache_mu);+ for (int t = 0; t < T; ++t) {+ const int64_t p = p0 + t;+ uint8_t* rows = dst != nullptr ? dst + (size_t) t * per : h_eg + ((size_t) li * eg_cap + t) * per;+ uint8_t* scales = rows + (size_t) cols * hd;+ if (ctoks[(size_t) p] < 0) { // a picture's token: no rows (zeros: its gate adds nothing)+ std::memset(rows, 0, per);+ continue;+ }+ int64_t prod[8];+ bool blocked = false;+ for (int s = 0; s < pk.eg_ngram; ++s) { // (the look-back stops at the start and at a picture)+ blocked = blocked || p < s || ctoks[(size_t) (p - s)] < 0;+ const int64_t tk = blocked ? pk.eg_pad : ctoks[(size_t) (p - s)];+ prod[s] = tk * pk.eg_mult[(size_t) li * pk.eg_ngram + s];+ }+ int64_t rolling = prod[0];+ for (int n = 1; n < pk.eg_ngram; ++n) {+ rolling ^= prod[n];+ for (int h = 0; h < pk.eg_heads; ++h) {+ const int c = (n - 1) * pk.eg_heads + h;+ const int64_t id = rolling % pk.eg_primes[(size_t) li * cols + c] + pk.eg_off[(size_t) li * cols + c];+ uint8_t* rd = rows + (size_t) c * hd;+ uint8_t* sd = scales + (size_t) c * (hd / 32);+ const uint64_t key = (uint64_t) id * 8 + (uint64_t) li;+ if (const uint8_t* hit = eg_cache.find(key)) {+ std::memcpy(rd, hit, (size_t) hd);+ std::memcpy(sd, hit + hd, (size_t) hd / 32);+ ++hits;+ continue;+ }+ const auto fi = first.find(key);+ if (fi != first.end()) {+ b.dups.push_back({fi->second, {rd, sd}});+ continue;+ }+ first.emplace(key, (int) b.reads.size());+ EgRead r;+ r.file = tw->file;+ r.dst = rd;+ r.scales = sd;+ r.key = key;+ if (mx8) {+ r.mx8 = true;+ r.n = (uint32_t) (hd / 32 * 33);+ r.off = tw->off + (uint64_t) id * r.n;+ } else if (q6k || iq4) { // (into the row's code bytes; its scale bytes unused)+ r.n = (uint32_t) (q6k ? hd / 256 * 210 : hd / 32 * 18);+ r.off = tw->off + (uint64_t) id * r.n;+ } else {+ r.n = (uint32_t) hd;+ r.off = tw->off + (uint64_t) id * hd;+ r.file2 = ts->file;+ r.n2 = (uint32_t) (hd / 32);+ r.off2 = ts->off + (uint64_t) id * (hd / 32);+ }+ b.reads.push_back(r);+ }+ }+ }+ }+ eg_rows_n += hits + (int64_t) b.reads.size() + (int64_t) b.dups.size();+ eg_hits_n += hits + (int64_t) b.dups.size();+ eg_reads_n += (int64_t) b.reads.size();+ eg_io.submit(b);+ }+ // b's rows all in place (any thread; a group's members may ask at once: one finishes, the others wait for it)+ void eg_finish(EgBatch& b) {+ if (b.post.load(std::memory_order_acquire) == 2) return;+ const int64_t t0 = now_ns();+ eg_io.wait(b);+ int want = 0;+ if (b.post.compare_exchange_strong(want, 1, std::memory_order_acq_rel)) {+ const int hd = pk.eg_hd;+ for (const auto& d : b.dups) {+ const EgRead& r = b.reads[(size_t) d.first];+ std::memcpy(d.second.first, r.dst, (size_t) hd);+ std::memcpy(d.second.second, r.scales, (size_t) hd / 32);+ }++ if (std::getenv("DS4_DBG_EG") && pk.eg_hd >= 32) { // DBG: a row whose IQ4_NL block scale is not finite++ for (size_t i = 0; i < b.reads.size(); ++i) {++ const EgRead& r = b.reads[i];++ for (int k = 0; k < hd / 32; ++k) {++ uint16_t hb;++ std::memcpy(&hb, r.dst + (size_t) k * 18, 2);++ if ((hb & 0x7c00) == 0x7c00)++ std::fprintf(stderr, "DBG eg bad scale: read %zu/%zu file=%d off=%llu n=%u block=%d half=%04x ok=%d\n", i, b.reads.size(),++ r.file, (unsigned long long) r.off, r.n, k, hb, (int) b.ok[i]);++ }++ }++ }+ {+ std::lock_guard<std::mutex> lk(eg_cache_mu);+ for (size_t i = 0; i < b.reads.size(); ++i) {+ const EgRead& r = b.reads[i];+ if (!b.ok[i]) { // (a failed read: zeros, its gate adds nothing; not kept)+ std::memset(r.dst, 0, (size_t) hd);+ std::memset(r.scales, 0, (size_t) hd / 32);+ continue;+ }+ if (uint8_t* c = eg_cache.put(r.key)) {+ std::memcpy(c, r.dst, (size_t) hd);+ std::memcpy(c + hd, r.scales, (size_t) hd / 32);+ }+ }+ }+ eg_io_ns += b.t1.load() - b.t0;+ b.live = false;+ b.post.store(2, std::memory_order_release);+ } else {+ while (b.post.load(std::memory_order_acquire) != 2) std::this_thread::yield();+ }+ eg_wait_ns += now_ns() - t0;+ }+ void eg_stats() {+ st.eg_rows = eg_rows_n.load();+ st.eg_hits = eg_hits_n.load();+ st.eg_reads = eg_reads_n.load();+ st.eg_io_ms = (double) eg_io_ns.load() / 1e6;+ std::lock_guard<std::mutex> lk(eg_cache_mu);+ st.eg_cached = (int64_t) eg_cache.size();+ st.eg_cache_cap = (int64_t) eg_cache.capacity();+ }+ // the chunked prompt path: rows of Engram layer `only` into dst, now (a helper thread, a chunk ahead)+ void engram_rows(int64_t p0, int T, int only, uint8_t* dst) {+ EgBatch b;+ eg_issue(p0, T, only, dst, b);+ eg_finish(b);+ }++ // ---- the CPU lane: a window's routed experts that live on the host, as entries (an expert and up to 8 of its+ // tokens, in rows slot0 ..) computed together: each node its column part of every matrix, from memory on that node,+ // in four rounds (the gate / up split-K pieces, each preparing its two input blocks of every token; each band's+ // sums / codebook / Hadamard / swiglu and that block of the down input, which both nodes then read whole; the down+ // pieces; the down bands). yd[slot] gets each row's expert output.+ static constexpr int kEntMax = kWin > kc::kExl3CpuMaxN ? kWin : kc::kExl3CpuMaxN;+ struct Ent {+ int e = 0, n = 0, slot0 = 0;+ int tok[kEntMax] = {};+ };+ // tokens an entry holds: the i-quant GGUF experts take a whole window's tokens of an expert in one call (the CPU+ // lane's exact multi-token rows, 7+ tokens an expert: decoded once for all), the others (MXFP4, EXL3) up to 8+ static int ent_cap(const Layer& l) {+ return l.lay.fmt == gg::kFmtMXFP4 && !gg::mx_experts(l.lay) ? kEntMax : kc::kExl3CpuMaxN;+ }+ void run_nodes(int n, const std::function<void(int)>* f) {+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->start(n, f[X]);+ pools[0]->help();+ for (int X = 0; X < NH; ++X) pools[(size_t) X]->wait();+ }++ // a GGUF's MXFP4 experts on the CPU lane, read from the file's mapping or their copy in RAM (mx_part_ptr): each+ // node its rows of every matrix - the gate / up rows [X n_ff / NH, ..) of each entry (the swiglu into its h row),+ // then (h whole) the down rows [X dim / NH, ..) into yd. A row's sum is the same whatever the split: the same bits+ // in a copy as in place. The device's arithmetic (gg::mx_rows: int8 activations rounded as the device rounds+ // them, exact integer sums a block; every CPU tier the same bits): each token's input quantized once, whatever+ // number of entries take it, and each job its band of h straight into the down's int8 input.+ // (ord: the order the entries are computed in - their slots, so the sums, stay the same; null: as they are)+ void cpu_experts_mx(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {+ if (!gg::mx_experts(l.lay)) return cpu_experts_iq(l, x, ents, ldx, ord);+ constexpr int RB = 64; // rows a job (whole blocks of h: RB % 32 == 0)+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();+ const size_t gr = gg::mx_row_bytes(E), dr = gg::mx_row_bytes(FF);+ const size_t xa = gg::mx_act_bytes(E), ha = gg::mx_act_bytes(FF);+ const bool band_q = FFh % 32 == 0; // (each job's rows whole blocks of h)+ const float lim = g.lim;+ static_assert(kc::kExl3CpuMaxN <= gg::kMxCpuMaxN, "a CPU-lane entry's tokens in one call");+ { // the experts read from the files (not in the RAM tier) that are not resident: all their pages asked for at+ // once, so the SSD reads them with large requests side by side while the first entries compute+ // (os::prefetch_cold: the resident ones are left alone - the hint walks every page of a range, which cost+ // more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)+ std::vector<std::pair<const void*, size_t>> want;+ for (const Ent& en : ents)+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});+ os::prefetch_cold(want);+ }+ // the int8 inputs in c_xh's bytes: token t's (t < kWin) at t xa, then the slots' h at (kWin + slot) ha; a token+ // index past the window (none on the decode or chunk paths) gets a row of its own after them+ uint8_t* xq = (uint8_t*) c_xh.data();+ uint8_t* hq = xq + (size_t) kWin * xa;+ uint8_t* xq_extra = hq + (size_t) kSlots * ha;+ auto x_act = [&](const Ent& en, int i) -> const void* {+ const int t = en.tok[i];+ return t < kWin ? xq + (size_t) t * xa : xq_extra + (size_t) (en.slot0 + i) * xa;+ };+ {+ bool done[kWin] = {};+ for (const Ent& en : ents)+ for (int i = 0; i < en.n; ++i) {+ const int t = en.tok[i];+ if (t < kWin && done[t]) continue;+ if (t < kWin) done[t] = true;+ gg::mx_quant(x + (size_t) t * ldx, E, (void*) x_act(en, i));+ }+ }+ std::function<void(int)> f[4];+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);+ const void* ap[gg::kMxCpuMaxN];+ float yg[gg::kMxCpuMaxN][RB], yu[gg::kMxCpuMaxN][RB];+ float *gp[gg::kMxCpuMaxN], *up[gg::kMxCpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ ap[i] = x_act(en, i);+ gp[i] = yg[i];+ up[i] = yu[i];+ }+ // (part X's rows, from where they are: a copy's part on node X, or the file's)+ gg::mx_rows(mx_part_ptr(l, en.e, 0, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, gp);+ gg::mx_rows(mx_part_ptr(l, en.e, 1, X), gr, E, r0 - X * FFh, r1 - X * FFh, ap, en.n, up);+ for (int i = 0; i < en.n; ++i) {+ float* h = band_q ? yg[i] : c_xhd.data() + (size_t) (en.slot0 + i) * FF + r0;+ for (int r = r0; r < r1; ++r) {+ float gv = yg[i][r - r0], uv = yu[i][r - r0];+ if (lim > 0.0f) {+ gv = std::min(gv, lim);+ uv = std::min(std::max(uv, -lim), lim);+ }+ h[r - r0] = gv / (1.0f + std::exp(-gv)) * uv;+ }+ if (band_q) gg::mx_quant_blocks(h, r0 / 32, r1 / 32, hq + (size_t) (en.slot0 + i) * ha, FF);+ }+ };+ run_nodes(ne * GB, f);+ if (!band_q)+ for (const Ent& en : ents)+ for (int i = 0; i < en.n; ++i)+ gg::mx_quant(c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF, hq + (size_t) (en.slot0 + i) * ha);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);+ const void* ap[gg::kMxCpuMaxN];+ float* yp[gg::kMxCpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ ap[i] = hq + (size_t) (en.slot0 + i) * ha;+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + r0;+ }+ gg::mx_rows(mx_part_ptr(l, en.e, 2, X), dr, FF, r0 - X * Eh, r1 - X * Eh, ap, en.n, yp);+ };+ run_nodes(ne * DB, f);+ }+ // ... the low-bit ones (ds4_iq's types): as above, through ggml-cpu's dots (gg::iq_cpu_*) - each token's input+ // quantized once to the weight type's activation (Q8_K) in its c_xh row's bytes, then (h whole) its h the same++#if defined(STRATA_METAL)++ // The CPU lane's cold reads were 78% of its time (a madvise(WILLNEED) of a range that is not in the page cache blocks while the SSD++ // reads it). Stale-router lookahead (no training): the GPU applies layer L+a's gate (+ bias, the router's own rule) to layer L's FFN++ // input (a gemv and a route a layer, queued behind the shared expert) and the host reads the guessed picks with the real ones; the++ // guessed CPU-lane experts whose pages are not in the page cache are read in on helper threads while this layer computes. Bytes per++ // layer capped (DS4_LA_MAX experts, default 4); DS4_LOOKAHEAD layers ahead (default 2; 0 off).++ static bool la_enabled() { return la_depth() > 0; }++ static int la_depth() {++ static const int d = std::getenv("DS4_LOOKAHEAD") ? std::atoi(std::getenv("DS4_LOOKAHEAD")) : 0; // (off by default until it is measured faster)++ return std::min(d, 2);++ }++ struct LaPool {++ std::mutex m;++ std::condition_variable cv;++ std::deque<std::pair<const void*, size_t>> q;++ LaPool() {++ const int nth = std::getenv("DS4_LA_THREADS") ? std::atoi(std::getenv("DS4_LA_THREADS")) : 3;++ for (int i = 0; i < nth; ++i)++ std::thread([this] {++ for (;;) {++ std::pair<const void*, size_t> r;++ {++ std::unique_lock<std::mutex> lk(m);++ cv.wait(lk, [&] { return !q.empty(); });++ r = q.front();++ q.pop_front();++ while (q.size() > 64) q.pop_front(); // (stale reads are worth nothing)++ }++ const uintptr_t a = (uintptr_t) r.first & ~(uintptr_t) 16383;++ const size_t len = r.second + ((uintptr_t) r.first - a);++ std::vector<unsigned char> vec((len + 16383) / 16384);++ bool cold = true;++ if (mincore((void*) a, len, (char*) vec.data()) == 0) {++ cold = false;++ for (unsigned char c : vec) if (!(c & 1)) { cold = true; break; }++ }++ if (cold) os::prefetch({r});++ }++ }).detach();++ }++ void push(const void* p, size_t n) {++ { std::lock_guard<std::mutex> lk(m); q.push_back({p, n}); }++ cv.notify_one();++ }++ };++ std::mutex la_mu;++ std::vector<std::vector<int>> la_guess; // per layer: the CPU-lane experts its last lookahead guessed++ long la_hit = 0, la_all = 0, la_issued = 0;++ void lookahead_prefetch(Layer& l, cudaEvent_t ev2, int la_n, const std::vector<Ent>& ents) {++ if (l.nm != 1) return;++ static LaPool* pool = nullptr;++ static bool reg = false;++ if (pool == nullptr) {++ la_guess.assign(L.size(), {});++ pool = new LaPool();++ }++ if (std::getenv("DS4_CPU_LOG") && !reg) {++ reg = true;++ static Impl* self = this;++ std::atexit([] { std::fprintf(stderr, "lookahead: %ld of %ld CPU-lane experts were guessed; %ld matrices queued\n", self->la_hit, self->la_all, self->la_issued); });++ }++ for (const Ent& en : ents) {++ ++la_all;++ const auto& gs = la_guess[(size_t) l.il];++ if (std::find(gs.begin(), gs.end(), en.e) != gs.end()) ++la_hit;++ }++ if (la_n <= 0) return;++ cudaEventSynchronize(ev2);++ static const int cap = std::getenv("DS4_LA_MAX") ? std::atoi(std::getenv("DS4_LA_MAX")) : 4;++ for (int a = 1; a <= la_n; ++a) {++ Layer& nx = L[(size_t) (l.il + a)];++ std::vector<int> cpu;++ for (int r = 0; r < nx.k_exp; ++r) {++ const int e = h_idx_a[(a - 1) * kPicks + r];++ if (e >= 0 && e < nx.n_exp && nx.slot[(size_t) e] < 0 && (int) cpu.size() < cap) cpu.push_back(e);++ }++ la_guess[nx.il] = cpu;++ for (int e : cpu)++ for (int r = 0; r < 3; ++r) { pool->push(mx_mat(nx, e, r), mx_part_bytes(nx, r)); ++la_issued; }++ }++ }++#endif+ void cpu_experts_iq(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx, const int* ord) {+ constexpr int RB = 64; // rows a job+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH;+ const int GB = (FFh + RB - 1) / RB, DB = (Eh + RB - 1) / RB, ne = (int) ents.size();+ gg::IqCpu fm;+ std::string ferr;+ if (!gg::iq_cpu_fmt(gg::expert_type(l.lay, 0), gg::expert_type(l.lay, 2), E, FF, fm, ferr) ||+ fm.act_bytes > (size_t) E * 4 || fm.h_bytes > (size_t) E * 4) {+ static std::atomic<bool> said{false};+ if (!said.exchange(true)) std::fprintf(stderr, "ds4: layer %d's CPU-lane experts: %s\n", l.il, ferr.c_str());+ for (const Ent& en : ents) std::memset(c_yd.data() + (size_t) en.slot0 * E, 0, (size_t) en.n * E * 4);+ return;+ }++ static double t_pre = 0, t_q1 = 0, t_r1 = 0, t_q2 = 0, t_r2 = 0; static long n_calls = 0; static bool reg = false;++ if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu_experts_iq phases (ms over %ld calls): prefetch %.1f, quant1 %.1f, gate_up %.1f, quant2 %.1f, down %.1f\n", n_calls, t_pre, t_q1, t_r1, t_q2, t_r2); }); }++ ++n_calls;++ double tp0 = now_ms();+ const float lim = g.lim;+ const double tp0 = now_ms();+ { // (os::prefetch_cold, as above: the resident ones are left alone - on Windows the hint walks every page of a+ // range, ~0.13 ms an IQ2_XXS expert, on the lane's critical path before the first entry computes.+ // off by default (the mmap path, owner PC: not faster in every pair); DS4_IQ_PF_COLD=1 turns it on)+ std::vector<std::pair<const void*, size_t>> want;+ for (const Ent& en : ents)+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))+ for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});+ static const bool pf_all = std::getenv("DS4_IQ_PF_COLD") == nullptr || std::atoi(std::getenv("DS4_IQ_PF_COLD")) == 0;++#if defined(__APPLE__) // (macOS: the cold reads of a call's matrices in parallel, and a page-cache census under DS4_CPU_LOG)++ if (pf_all && std::getenv("DS4_CPU_LOG")) { // how many of the experts' matrices are in the page cache (mincore, the first 64 pages)++ static long cold_n = 0, warm_n = 0; static double cold_ms = 0, warm_ms = 0; static bool reg2 = false;++ if (!reg2) { reg2 = true; std::atexit([] { std::fprintf(stderr, "prefetch: %ld matrices cold (%.1f ms), %ld warm (%.1f ms)\n", cold_n, cold_ms, warm_n, warm_ms); }); }++ for (const auto& w : want) {++ const uintptr_t a = (uintptr_t) w.first & ~(uintptr_t) 16383;++ unsigned char vec[64];++ const size_t len = std::min<size_t>(w.second, 64 * 16384);++ bool cold = false;++ if (mincore((void*) a, len, (char*) vec) == 0) for (size_t i = 0; i < (len + 16383) / 16384; ++i) if (!(vec[i] & 1)) cold = true;++ const double t0 = now_ms();++ os::prefetch({w});++ (cold ? cold_ms : warm_ms) += now_ms() - t0;++ (cold ? cold_n : warm_n)++;++ }++ } else if (pf_all && want.size() > 1 && std::getenv("DS4_SERIAL_PREFETCH") == nullptr) {++ // (a madvise(WILLNEED) of a cold range blocks while the SSD reads it: queue depth 1 reads 3.9 GB/s; the matrices of++ // all the call's experts at once keep the SSD busy)++ std::function<void(int)> pf[4];++ for (int X = 0; X < NH; ++X) pf[X] = [&](int j) { os::prefetch({want[(size_t) j]}); };++ run_nodes((int) want.size(), pf);++ } else+ if (pf_all) os::prefetch(want);+ else os::prefetch_cold(want);++#else++ if (pf_all) os::prefetch(want);++ else os::prefetch_cold(want);++#endif+ }++ double tp1 = now_ms(); t_pre += tp1 - tp0;+ // the activations in c_xh's bytes, E 4 apart: token t's (t < kWin) at row t - quantized once, whatever number of+ // entries take it - then a row a slot (kWin + slot): a token past the window's input (none on the decode or+ // chunk paths), then (x no longer needed) the slot's h+ uint8_t* aq = (uint8_t*) c_xh.data();+ auto x_act = [&](const Ent& en, int i) -> uint8_t* {+ const int t = en.tok[i];+ return aq + (size_t) (t < kWin ? t : kWin + en.slot0 + i) * E * 4;+ };+ {+ bool done[kWin] = {};+ for (const Ent& en : ents)+ for (int i = 0; i < en.n; ++i) {+ const int t = en.tok[i];+ if (t < kWin && done[t]) continue;+ if (t < kWin) done[t] = true;+ gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));+ }+ }- const double tp1 = now_ms();++ double tq1 = now_ms(); t_q1 += tq1 - tp1;+ std::function<void(int)> f[4];+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / GB] : j / GB)];+ const int r0 = X * FFh + (j % GB) * RB, r1 = std::min(X * FFh + FFh, r0 + RB);+ const void* ap[kEntMax];+ float* hp[kEntMax];+ for (int i = 0; i < en.n; ++i) {+ ap[i] = x_act(en, i);+ hp[i] = c_xhd.data() + (size_t) (en.slot0 + i) * FF + (size_t) X * FFh; // (part X's rows of h)+ }+ // (part X's rows, from where they are: a copy's part on node X, or the file's - whole rows, so any+ // block size of the type)+ gg::iq_cpu_gate_up(fm, mx_part_ptr(l, en.e, 0, X), mx_part_ptr(l, en.e, 1, X), ap, en.n, hp,+ r0 - X * FFh, r1 - X * FFh, lim);+ };+ run_nodes(ne * GB, f);- const double tp2 = now_ms();++ double tr1 = now_ms(); t_r1 += tr1 - tq1;+ for (const Ent& en : ents)+ for (int i = 0; i < en.n; ++i)+ gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,+ aq + (size_t) (kWin + en.slot0 + i) * E * 4);- const double tp3 = now_ms();++ double tq2 = now_ms(); t_q2 += tq2 - tr1;+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];+ const int r0 = X * Eh + (j % DB) * RB, r1 = std::min(X * Eh + Eh, r0 + RB);+ const void* hp[kEntMax];+ float* yp[kEntMax];+ for (int i = 0; i < en.n; ++i) {+ hp[i] = aq + (size_t) (kWin + en.slot0 + i) * E * 4;+ yp[i] = c_yd.data() + (size_t) (en.slot0 + i) * E + (size_t) X * Eh; // (part X's rows of it)+ }+ gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);+ };+ run_nodes(ne * DB, f);++ t_r2 += now_ms() - tq2;+ { // DS4_PROF: where the CPU lane's time goes (act prep + prefetch, gate/up, h quant, down), summed+ static const bool on = std::getenv("DS4_PROF") != nullptr;+ if (on) {+ static double acc[6] = {};+ const double te = now_ms();- acc[0] += tp1 - tp0; acc[1] += tp2 - tp1; acc[2] += tp3 - tp2; acc[3] += te - tp3; acc[4] += 1; acc[5] += ne;++ acc[0] += tq1 - tp0; acc[1] += tr1 - tq1; acc[2] += tq2 - tr1; acc[3] += te - tq2; acc[4] += 1; acc[5] += ne;+ if (((long long) acc[4]) % 1000 == 0)+ std::fprintf(stderr, "ds4 prof CPU lane (IQ rows) per call: n %.0f, entries/call %.2f, prep %.3f ms, gate+up %.3f ms, h-quant %.3f ms, down %.3f ms\n", acc[4], acc[5] / acc[4], acc[0] / acc[4], acc[1] / acc[4], acc[2] / acc[4], acc[3] / acc[4]);+ }+ }+ }++ // the window's CPU-lane experts made ready (and pinned) in the store for the length of the call+ struct StoreHold {+ Impl* m = nullptr;+ int layer = 0;+ std::vector<int> es, sl;+ std::vector<char> ok;+ // lazy: the reads are started here and waited for by wait_all() / wait_expert(); the destructor releases+ StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {+ if (!m_->store) return;+ m = m_;+ layer = l.il;+ for (const Ent& en : ents)+ if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);+ m->store->begin(layer, es.data(), (int) es.size(), ok, sl);+ }+ int idx(int e) const {+ for (size_t i = 0; i < es.size(); ++i)+ if (es[i] == e) return (int) i;+ return -1;+ }+ // expert e needs no waiting: not a store expert (RAM copy) or its reads are done+ bool ready(int e) const {+ if (!m) return true;+ const int i = idx(e);+ return i < 0 || m->store->loaded(sl[(size_t) i]);+ }+ void wait_expert(int e) {+ if (!m) return;+ const int i = idx(e);+ if (i >= 0) m->store->wait_slots(sl, ok, i, i + 1);+ }+ void wait_all() {+ if (m) m->store->wait_slots(sl, ok, 0, (int) es.size());+ }+ ~StoreHold() {+ if (m) m->store->release(layer, es.data(), (int) es.size(), ok);+ }+ };+ // DS4_MISS_SPLIT=0: the old order (wait for every read, then compute); default: the entries whose reads are done+ // computed first, the others each as its read lands (the same bits: every entry's rows are independent)+ void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,+ const int* ord = nullptr) {+ if (ldx == 0) ldx = (size_t) g.dim;+ if (l.lay.fmt == gg::kFmtMXFP4) {+ StoreHold hold(this, l, ents);+ static const bool split = std::getenv("DS4_MISS_SPLIT") == nullptr || std::atoi(std::getenv("DS4_MISS_SPLIT")) != 0;+ if (!split || ord != nullptr || !hold.m || ents.size() < 2) {+ hold.wait_all();+ return cpu_experts_mx(l, x, ents, ldx, ord);+ }+ std::vector<Ent> hot, cold;+ for (const Ent& en : ents) (hold.ready(en.e) ? hot : cold).push_back(en);+ if (cold.empty()) {+ hold.wait_all();+ return cpu_experts_mx(l, x, ents, ldx, ord);+ }+ for (const Ent& en : hot) hold.wait_expert(en.e); // (done reading: only the bookkeeping)+ if (!hot.empty()) cpu_experts_mx(l, x, hot, ldx, nullptr);+ // the ones that came in meanwhile first, then the rest in the order they land+ std::vector<char> done(cold.size(), 0);+ for (size_t left = cold.size(); left > 0;) {+ size_t pick = cold.size();+ for (size_t i = 0; i < cold.size(); ++i)+ if (!done[i] && hold.ready(cold[i].e)) {+ pick = i;+ break;+ }+ if (pick == cold.size())+ for (size_t i = 0; i < cold.size(); ++i)+ if (!done[i]) {+ pick = i;+ break;+ }+ hold.wait_expert(cold[pick].e);+ std::vector<Ent> one(1, cold[pick]);+ cpu_experts_mx(l, x, one, ldx, nullptr);+ done[pick] = 1;+ --left;+ }+ return;+ }+ const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;+ const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;+ const int ne = (int) ents.size();+ const float lim = g.lim;+ auto part = [&](int e, int X, int r) {+ const uint8_t* b = l.hh[(size_t) e][(size_t) X];+ kc::Exl3Part p;+ p.trellis = (const uint32_t*) (b + l.hl.trellis[r]);+ p.suh = (const uint16_t*) (b + l.hl.suh[r]);+ p.svh = (const uint16_t*) (b + l.hl.svh[r]);+ p.k = l.hl.k[r];+ p.n = l.hl.n[r];+ p.K = l.lay.K[r];+ return p;+ };+ auto xh = [&](int X, int sl, int r) { return c_xh.data() + ((size_t) (X * kSlots + sl) * 2 + r) * E; };+ auto xsb = [&](int X, int sl, int r) { return c_xsb.data() + ((size_t) (X * kSlots + sl) * 2 + r) * EB; };+ auto acc = [&](int X, int sl, int r, int q) {+ return c_acc.data() + (((size_t) (X * kSlots + sl) * 2 + r) * GQ + q) * FFh;+ };+ auto xhd = [&](int sl) { return c_xhd.data() + (size_t) sl * FF; };+ auto xsd = [&](int sl) { return c_xsd.data() + (size_t) sl * FB; };+ auto accd = [&](int X, int sl, int q) { return c_accd.data() + ((size_t) (X * kSlots + sl) * DQ + q) * Eh; };+ std::function<void(int)> f[4];+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (j / (2 * GQ))];+ const int r = (j / GQ) % 2, q = j % GQ;+ const kc::Exl3Part p = part(en.e, X, r);+ const float* xp[kc::kExl3CpuMaxN];+ float* ap[kc::kExl3CpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ const int sl = en.slot0 + i;+ const float* xt = x + (size_t) en.tok[i] * ldx;+ for (int b = 2 * q; b < 2 * q + 2; ++b)+ xsb(X, sl, r)[b] = kc::exl3_cpu_prepare_block(p.suh + b * 128, xt + b * 128, xh(X, sl, r) + b * 128);+ xp[i] = xh(X, sl, r);+ ap[i] = acc(X, sl, r, q);+ }+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 16, (q + 1) * 16, ap[0]);+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 16, (q + 1) * 16, ap);+ };+ run_nodes(ne * 2 * GQ, f);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (j / GB)];+ const int b = j % GB;+ const kc::Exl3Part pg = part(en.e, X, 0), pu = part(en.e, X, 1), pd = part(en.e, X, 2);+ for (int i = 0; i < en.n; ++i) {+ const int sl = en.slot0 + i;+ float y[2][128], a[128], hb[128];+ for (int r = 0; r < 2; ++r) {+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;+ for (int q = 0; q < GQ; ++q) {+ const float* aq = acc(X, sl, r, q) + b * 128;+ for (int c = 0; c < 128; ++c) a[c] += aq[c];+ }+ double xs = 0.0;+ for (int k = 0; k < EB; ++k) xs += xsb(X, sl, r)[k];+ kc::exl3_cpu_band_out(r == 0 ? pg : pu, a, (float) xs, b * 128, y[r]);+ }+ for (int c = 0; c < 128; ++c) {+ float gv = y[0][c], uv = y[1][c];+ if (lim > 0.0f) {+ gv = std::min(gv, lim);+ uv = std::min(std::max(uv, -lim), lim);+ }+ hb[c] = gv / (1.0f + std::exp(-gv)) * uv;+ }+ const int gb = X * GB + b;+ xsd(sl)[gb] = kc::exl3_cpu_prepare_block(pd.suh + gb * 128, hb, xhd(sl) + gb * 128);+ }+ };+ run_nodes(ne * GB, f);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (j / DQ)];+ const int q = j % DQ;+ const kc::Exl3Part p = part(en.e, X, 2);+ const float* xp[kc::kExl3CpuMaxN];+ float* ap[kc::kExl3CpuMaxN];+ for (int i = 0; i < en.n; ++i) {+ xp[i] = xhd(en.slot0 + i);+ ap[i] = accd(X, en.slot0 + i, q);+ }+ if (en.n == 1) kc::exl3_cpu_rows_acc(p, xp[0], q * 8, (q + 1) * 8, ap[0]);+ else kc::exl3_cpu_rows_acc_n(p, xp, en.n, q * 8, (q + 1) * 8, ap);+ };+ run_nodes(ne * DQ, f);+ for (int X = 0; X < NH; ++X)+ f[X] = [&, X](int j) {+ const Ent& en = ents[(size_t) (j / DB)];+ const int b = j % DB;+ const kc::Exl3Part pd = part(en.e, X, 2);+ for (int i = 0; i < en.n; ++i) {+ const int sl = en.slot0 + i;+ float a[128];+ for (int c = 0; c < 128; ++c) a[c] = 0.0f;+ for (int q = 0; q < DQ; ++q) {+ const float* aq = accd(X, sl, q) + b * 128;+ for (int c = 0; c < 128; ++c) a[c] += aq[c];+ }+ double xs = 0.0;+ for (int k = 0; k < FB; ++k) xs += xsd(sl)[k];+ kc::exl3_cpu_band_out(pd, a, (float) xs, b * 128, c_yd.data() + (size_t) sl * E + (size_t) X * Eh + b * 128);+ }+ };+ run_nodes(ne * DB, f);+ }++ // ---- the expert cache. A swap moves a CPU-lane expert (`in`) into the layer's spare VRAM slot and the VRAM expert it+ // replaces (`out`) into the spare host parts - both copies of weights that do not change, made while the model runs+ // (the copy stream, the host's repacking) - and cache_tick switches the tables between windows, so the device's+ // table and the host's view of a layer always agree within a window. In place (file_experts) `out` needs no copy:+ // the CPU lane reads it from the file again.+ void ram_loop() { // the RAM tier's moves in: an expert read from the file into a free slot+ for (;;) {+ RamSwap rs;+ {+ std::unique_lock<std::mutex> lk(sw_mu);+ ram_cv.wait(lk, [&] { return quit.load() || !ram_queue.empty(); });+ if (quit.load()) return;+ rs = ram_queue.front();+ ram_queue.pop_front();+ }+ const Layer& l = L[(size_t) rs.layer];+ if (read_mx_file(l, rs.in, ram_slots[(size_t) rs.slot])) {+ for (int r = 0; r < 3; ++r) // (one copy of it in RAM)+ os::advise(pk.fds[(size_t) l.mx_file[r]], l.mx_off[r] + (uint64_t) rs.in * l.mx_stride[r],+ mx_part_bytes(l, r), false);+ } else {+ rs.in = -1; // (failed: the slot goes back)+ }+ std::lock_guard<std::mutex> lk(sw_mu);+ ram_ready.push_back(rs);+ }+ }++ void cache_loop(SwapBufs& B) {+ for (;;) {+ Swap sw;+ {+ std::unique_lock<std::mutex> lk(sw_mu);+ sw_cv.wait(lk, [&] { return quit.load() || !sw_queue.empty(); });+ if (quit.load()) return;+ sw = sw_queue.front();+ sw_queue.pop_front();+ }+ Layer& l = L[(size_t) sw.layer];+ if (l.lay.fmt == gg::kFmtMXFP4) { // in: from the file; out: nothing to keep (the file has it)+ swap_in_mx(l, sw, B);+ std::lock_guard<std::mutex> lk(sw_mu);+ sw_ready.push_back(sw);+ continue;+ }+ parts_to_blob(l, l.hh[(size_t) sw.in], B.a);+ if (l.nm == 1) {+ const Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaStream_t cs = B.st[(size_t) l.dev];+ cudaMemcpyAsync(l.vram_base + (size_t) l.spare_vram * l.blob, B.a, l.blob, cudaMemcpyHostToDevice, cs);+ if (!file_experts)+ cudaMemcpyAsync(B.b, l.vram_base + (size_t) l.slot[(size_t) sw.out] * l.blob, l.blob,+ cudaMemcpyDeviceToHost, cs);+ cudaStreamSynchronize(cs);+ } else { // a group: each member's part in, each member's part out+ for (int m = 0; m < l.nm; ++m) {+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaStream_t cs = B.st[(size_t) x.dev];+ blob_to_half(l, B.a, m, B.h);+ cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * x.hblob, B.h, x.hblob, cudaMemcpyHostToDevice, cs);+ cudaStreamSynchronize(cs);+ if (file_experts) continue;+ cudaMemcpyAsync(B.h, x.vram_base + (size_t) l.slot[(size_t) sw.out] * x.hblob, x.hblob,+ cudaMemcpyDeviceToHost, cs);+ cudaStreamSynchronize(cs);+ half_to_blob(l, B.h, m, B.b);+ }+ }+ if (!file_experts)+ for (int X = 0; X < NH; ++X) blob_to_part(l, B.b, X, l.spare_host[(size_t) X]);+ if (check_swaps) { // DS4_CHECK_SWAPS: both moves against the model file, byte for byte+ std::vector<uint8_t> reg, blob(l.blob), part(l.hl.bytes);+ bool ok = true;+ for (int pass = 0; pass < (file_experts ? 1 : 2); ++pass) { // (in place: no host copy of `out`)+ const int e = pass == 0 ? sw.in : sw.out;+ reg.resize(l.region_bytes[(size_t) e]);+ pk.read_at(l.region_file[(size_t) e], l.region_off[(size_t) e], reg.size(), reg.data());+ if (pass == 0) {+ repack(l, reg.data(), blob.data());+ std::vector<uint8_t> dev(l.blob);+ if (l.nm == 1)+ cudaMemcpy(dev.data(), l.vram_base + (size_t) l.spare_vram * l.blob, l.blob, cudaMemcpyDeviceToHost);+ else+ for (int m = 0; m < l.nm; ++m) { // (the members' parts, put together again)+ std::vector<uint8_t> h(member(l, m).hblob);+ cudaSetDevice(devs[(size_t) member(l, m).dev].id);+ cudaMemcpy(h.data(), member(l, m).vram_base + (size_t) l.spare_vram * member(l, m).hblob, member(l, m).hblob,+ cudaMemcpyDeviceToHost);+ half_to_blob(l, h.data(), m, dev.data());+ }+ for (int r = 0; r < 3; ++r)+ ok = ok && std::memcmp(dev.data() + l.lay.trellis[r], blob.data() + l.lay.trellis[r], l.piece_bytes[r][0]) == 0 &&+ std::memcmp(dev.data() + l.lay.suh[r], blob.data() + l.lay.suh[r], l.piece_bytes[r][1]) == 0 &&+ std::memcmp(dev.data() + l.lay.svh[r], blob.data() + l.lay.svh[r], l.piece_bytes[r][2]) == 0;+ } else {+ for (int X = 0; X < NH; ++X) {+ repack_part(l, reg.data(), X, part.data());+ for (int r = 0; r < 3; ++r) {+ const size_t tb = (size_t) (l.hl.k[r] / 16) * (l.hl.n[r] / 16) * 32 * l.lay.K[r];+ ok = ok && std::memcmp(part.data() + l.hl.trellis[r], l.spare_host[(size_t) X] + l.hl.trellis[r], tb) == 0 &&+ std::memcmp(part.data() + l.hl.suh[r], l.spare_host[(size_t) X] + l.hl.suh[r], (size_t) l.hl.k[r] * 2) == 0 &&+ std::memcmp(part.data() + l.hl.svh[r], l.spare_host[(size_t) X] + l.hl.svh[r], (size_t) l.hl.n[r] * 2) == 0;+ }+ }+ }+ }+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out: %s\n", l.il, sw.in, sw.out,+ ok ? "both match the model file" : "MISMATCH");+ }+ std::lock_guard<std::mutex> lk(sw_mu);+ sw_ready.push_back(sw);+ }+ }++ // a GGUF's swap: expert sw.in from the file (copy_mode: its copy) into the spare VRAM slot (each member its part);+ // the expert it replaces goes back to the CPU lane, which reads it from the file's mapping - or (copy_mode, sw.ram)+ // from the layer's spare RAM slot, which it is copied into from VRAM here+ void swap_in_mx(Layer& l, Swap& sw, SwapBufs& B) {+ uint8_t *sw_a = B.a, *sw_b = B.b, *sw_h = B.h; // (this thread's)+ const std::vector<cudaStream_t>& copy_st = B.st;+ bool ok = copy_mode ? read_mx(l, sw.in, sw_a) : read_mx_file(l, sw.in, sw_a); // (a copy holds its slot till+ // this swap's switch)+ for (int m = 0; m < l.nm && ok; ++m) {+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaStream_t cs = copy_st[(size_t) x.dev];+ if (l.nm > 1) blob_to_half(l, sw_a, m, sw_h);+ ok = cudaMemcpyAsync(x.vram_base + (size_t) l.spare_vram * unit(l), l.nm > 1 ? sw_h : sw_a, unit(l),+ cudaMemcpyHostToDevice, cs) == cudaSuccess &&+ cudaStreamSynchronize(cs) == cudaSuccess;+ }+ if (sw.ram >= 0) { // `out` from its VRAM slot (each member its part) into the spare RAM slot+ bool got = ok;+ for (int m = 0; m < l.nm && got; ++m) {+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaStream_t cs = copy_st[(size_t) x.dev];+ got = cudaMemcpyAsync(l.nm > 1 ? sw_h : sw_b, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l),+ unit(l), cudaMemcpyDeviceToHost, cs) == cudaSuccess &&+ cudaStreamSynchronize(cs) == cudaSuccess;+ if (got && l.nm > 1) half_to_blob(l, sw_h, m, sw_b);+ }+ if (got) blob_to_ram(l, sw_b, sw.ram);+ else sw.ram = -1; // (back to the file's pages then)+ } else if (ok && store && l.nm == 1 && l.lay.fmt == gg::kFmtMXFP4 && strata::ds4::store_feature_default(env_tri("DS4_STORE_PUT"), defaults_026())) {+ // `out` goes back to the CPU lane: its blob is in VRAM, so it is copied into a store slot now - the lane would otherwise read it from+ // the SSD and wait for it (1-2 misses a token on the 251 GB R9700 box). DS4_STORE_PUT=0: off.+ uint8_t* dst = store->put_begin(l.il, sw.out);+ if (dst != nullptr) {+ const Layer& x = member(l, 0);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaStream_t cs = copy_st[(size_t) x.dev];+ const bool got = cudaMemcpyAsync(dst, x.vram_base + (size_t) l.slot[(size_t) sw.out] * unit(l), l.blob, cudaMemcpyDeviceToHost, cs) ==+ cudaSuccess &&+ cudaStreamSynchronize(cs) == cudaSuccess;+ store->put_done(l.il, sw.out, got);+ }+ }+ if (check_swaps) { // DS4_CHECK_SWAPS: the slot (and `out`'s RAM copy) against the model file, byte for byte+ std::vector<uint8_t> dev(l.blob), h(unit(l)), file(l.blob);+ for (int m = 0; m < l.nm && ok; ++m) {+ const Layer& x = member(l, m);+ cudaSetDevice(devs[(size_t) x.dev].id);+ cudaMemcpy(l.nm > 1 ? h.data() : dev.data(), x.vram_base + (size_t) l.spare_vram * unit(l), unit(l),+ cudaMemcpyDeviceToHost);+ if (l.nm > 1) half_to_blob(l, h.data(), m, dev.data());+ }+ ok = ok && read_mx_file(l, sw.in, file.data());+ for (int r = 0; r < 3 && ok; ++r)+ ok = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;+ bool ok_out = true;+ if (sw.ram >= 0) {+ ok_out = read_mx_file(l, sw.out, file.data());+ ram_to_blob(l, sw.ram, dev.data());+ for (int r = 0; r < 3 && ok_out; ++r)+ ok_out = std::memcmp(dev.data() + l.lay.trellis[r], file.data() + l.lay.trellis[r], mx_part_bytes(l, r)) == 0;+ }+ std::fprintf(stderr, "ds4 cache: layer %d expert %d in, %d out%s: %s\n", l.il, sw.in, sw.out,+ sw.ram >= 0 ? " (into RAM)" : "", ok && ok_out ? "matches the model file" : "MISMATCH");+ } else if (!ok) {+ std::fprintf(stderr, "ds4 cache: layer %d expert %d: the move into VRAM failed\n", l.il, sw.in);+ }+ }++ // after a chunked prompt: the expert cache catches up with what the prompt picked at once - a window prompt gets+ // its swaps one tick at a time over its hundreds of windows, a chunked one has a single tick - so the decode that+ // follows finds the hot experts in VRAM. Rounds of the ordinary swap rule (the hottest CPU-lane pick of each layer+ // in, its coldest VRAM expert out) until nothing is worth moving or DS4_SETTLE_MS (6000) is spent+ void cache_settle() {+ if (!o.expert_cache || cache_ths.empty()) return;+ static const double budget = env_f("DS4_SETTLE_MS", 6000.0f);+ if (budget <= 0.0) return;+ const double t0 = now_ms();+ const int64_t s0 = st.swaps;+ while (now_ms() - t0 < budget) {+ for (Layer& l : L) {+ if (l.mtp || l.spare_vram < 0 || l.swap_pending) continue;+ l.cpu_hits.clear();+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0 && l.heat[(size_t) e] >= 2.0f) l.cpu_hits.push_back(e);+ }+ cache_tick_swaps(0);+ if (sw_inflight == 0) break; // (nothing queued: every layer's best CPU-lane expert is not hotter than its coldest VRAM one)+ std::this_thread::sleep_for(std::chrono::milliseconds(1));+ }+ // The RAM store's preload after a chunked prompt is off by default: measured on Strix Halo (code fixture, prompt ++ // 256 tokens, 3 runs a side) it costs the prompt 4.7 s and the decode that follows gains nothing (13.9/14.4 s with it+ // against 14.2 s without). The swaps above are what the decode actually uses; DS4_SETTLE_STORE=1 restores it.+ if (store && env_f("DS4_SETTLE_STORE", strata::ds4::settle_store_default(defaults_026())) != 0.0f) { // the RAM store too: a window prompt fills it through the CPU+ // lane's picks, a chunked one streams past it - the hottest CPU-lane experts of the prompt into its free slots+ // (copy_mode: an expert already in the CPU lane's RAM copy gains nothing from a second copy of the same bytes -+ // measured on the R9700 12 GB cell: 4152 such experts, 40 GB read at 6.4 GB/s, 6.2 s of an 8K prompt, 0.0 store+ // hits in the decode after it; the VRAM swaps above already read them from the copy)+ std::vector<std::pair<float, std::pair<int, int>>> hot;+ for (const Layer& l : L) {+ if (l.mtp) continue;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0 && !(strata::ds4::settle_skips_copy_experts(defaults_026()) && copy_mode && !l.ram.empty() && l.ram[(size_t) e] >= 0) &&+ l.heat[(size_t) e] >= 4.0f) hot.push_back({l.heat[(size_t) e], {l.il, e}});+ }+ std::sort(hot.begin(), hot.end(), [](const auto& a, const auto& b) { return a.first > b.first; });+ std::vector<std::pair<int, int>> list;+ for (const auto& h : hot) {+ if (list.size() >= store->slots_max()) break;+ list.push_back(h.second);+ }+ const double ts = now_ms();+ const size_t n = store->preload(list, 0);+ if (o.verbose) std::fprintf(stderr, "ds4: RAM store: %zu hot experts read after the prompt (%.0f ms)\n", n, now_ms() - ts);+ }+ if (o.verbose && st.swaps > s0)+ std::fprintf(stderr, "ds4: expert cache: %lld swaps after the prompt (%.0f ms)\n", (long long) (st.swaps - s0),+ now_ms() - t0);+ }+ void cache_tick(int T) {+ cache_tick_swaps(T);+ store_sync();+ store_watch();+ }+ // the store gives RAM back when the PC runs short (the owner opened something): below 4 GB free, a 0.7 GB chunk at a time+ // until 5 GB are free again (checked every 16 windows); it grows again by itself once 8 GB beside its chunk are free+ void store_watch() {+ if (!store) return;+ static int tick = 0;+ if (++tick % 16 != 0) return;+ const uint64_t avail = ram_avail();+ if (avail == ~uint64_t{0} || avail >= (4ull << 30)) return;+ const size_t chunks = 1 + (size_t) (((5ull << 30) - avail) / (700ull << 20));+ const size_t n = store->shrink_chunks(chunks);+ if (n > 0) std::fprintf(stderr, "ds4: RAM store: %.1f GB free: gave back %zu chunks (%zu slots left)\n", avail / 1e9, n, store->slots_allocated());+ }+ void cache_tick_swaps(int T) {+ if (!o.expert_cache || cache_ths.empty()) return;+ std::vector<Swap> ready;+ std::vector<RamSwap> rready;+ {+ std::lock_guard<std::mutex> lk(sw_mu);+ ready.swap(sw_ready);+ rready.swap(ram_ready);+ }+ for (const RamSwap& rs : rready) { // (nothing reads the tier between windows)+ --ram_inflight;+ Layer& l = L[(size_t) rs.layer];+ if (rs.in < 0 || l.ram[(size_t) rs.in] >= 0 || l.slot[(size_t) rs.in] >= 0) {+ ram_free.push_back(rs.slot);+ continue;+ }+ l.ram[(size_t) rs.in] = rs.slot;+ if (rs.out >= 0 && l.ram[(size_t) rs.out] >= 0) {+ ram_free.push_back(l.ram[(size_t) rs.out]);+ l.ram[(size_t) rs.out] = -1;+ }+ ++st.ram_moves;+ }+ for (const Swap& sw : ready) {+ Layer& l = L[(size_t) sw.layer];+ const int old_slot = l.slot[(size_t) sw.out];+ const std::array<uint8_t*, 4> old_parts = l.hh[(size_t) sw.in];+ if (copy_mode) { // (`out` takes the RAM place `in` leaves: through the spare, which `in`'s slot becomes)+ const int s_in = l.ram[(size_t) sw.in];+ l.ram[(size_t) sw.in] = -1;+ if (sw.ram >= 0) {+ l.ram[(size_t) sw.out] = sw.ram;+ copy_spare[(size_t) l.il] = s_in;+ } else if (s_in >= 0) { // (the move into RAM failed: `out` back to the file, one copy fewer)+ --ram_held;+ if (copy_spare[(size_t) l.il] < 0) copy_spare[(size_t) l.il] = s_in;+ }+ } else if (!l.ram.empty() && l.ram[(size_t) sw.in] >= 0) { // (in VRAM now: its RAM slot for the next hottest)+ ram_free.push_back(l.ram[(size_t) sw.in]);+ l.ram[(size_t) sw.in] = -1;+ }+ l.slot[(size_t) sw.in] = l.spare_vram;+ if (!l.zc.empty() && l.zc[(size_t) sw.in]) { // (VRAM's now: not the store's to compute any more)+ l.zc[(size_t) sw.in] = 0;+ if (l.zc_tab != nullptr) { // (the device's table too: its entry pointed into the slot the store now reuses - when `in` left VRAM later,+ // zc_stage copied whatever lay there, another expert's blob, into the plan)+ cudaSetDevice(devs[0].id);+ cudaMemsetAsync(l.zc_tab + sw.in, 0, 8, devs[0].st);+ }+ store->unmark(l.il, sw.in);+ }+ if (l.nm == 1) set_vm(l, sw.in, l.spare_vram);+ l.hh[(size_t) sw.in] = {nullptr, nullptr, nullptr, nullptr};+ l.host[(size_t) sw.in] = nullptr;+ l.slot[(size_t) sw.out] = -1;+ if (l.lay.fmt == gg::kFmtMXFP4) { // (the file has it)+ l.host[(size_t) sw.out] = (uint8_t*) mx_mat(l, sw.out, 0);+ } else if (file_experts) { // (back to its pages in the file)+ l.hh[(size_t) sw.out] = {file_part(l, sw.out), nullptr, nullptr, nullptr};+ l.host[(size_t) sw.out] = l.hh[(size_t) sw.out][0];+ } else {+ l.hh[(size_t) sw.out] = l.spare_host;+ l.host[(size_t) sw.out] = l.spare_host[0];+ l.spare_host = old_parts;+ }+ l.spare_vram = old_slot;+ for (int m = 0; m < l.nm; ++m) {+ Layer& x = member(l, m);+ const Dev& dm = devs[(size_t) x.dev];+ l.tab_pin[2 * m] = (unsigned long long) (x.vram_base + (size_t) l.slot[(size_t) sw.in] * unit(x));+ l.tab_pin[2 * m + 1] = 0ull;+ cudaSetDevice(dm.id); // (stream order: before the next window)+ cudaMemcpyAsync(x.vram_tab + sw.in, &l.tab_pin[2 * m], 8, cudaMemcpyHostToDevice, dm.st);+ cudaMemcpyAsync(x.vram_tab + sw.out, &l.tab_pin[2 * m + 1], 8, cudaMemcpyHostToDevice, dm.st);+ }+ l.swap_pending = false;+ --sw_inflight;+ ++st.swaps;+ }+ decay_tokens += T;+ if (decay_tokens >= 256) { // older use counts less: x0.9 every 256 tokens+ decay_tokens = 0;+ for (Layer& l : L)+ for (float& h : l.heat) h *= c_decay;+ }+ bool queued = false, rqueued = false;+ for (Layer& l : L) {+ if (l.cpu_hits.empty()) continue;+ std::vector<int> hits;+ hits.swap(l.cpu_hits);+ int best = -1, vin = -1;+ for (int e : hits)+ if (l.slot[(size_t) e] < 0 && (best < 0 || l.heat[(size_t) e] > l.heat[(size_t) best])) best = e;+ if (best >= 0 && l.spare_vram >= 0 && !l.swap_pending && sw_inflight < c_inflight) { // VRAM: the hottest in+ int cold = -1;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] >= 0 && (cold < 0 || l.heat[(size_t) e] < l.heat[(size_t) cold])) cold = e;+ if (cold >= 0 && l.heat[(size_t) best] > l.heat[(size_t) cold] * c_margin + c_bonus) {+ l.swap_pending = true;+ ++sw_inflight;+ vin = best;+ Swap sw{l.il, best, cold};+ if (copy_mode && l.ram[(size_t) best] >= 0) sw.ram = copy_spare[(size_t) l.il]; // (-1: none)+ std::lock_guard<std::mutex> lk(sw_mu);+ sw_queue.push_back(sw);+ queued = true;+ }+ }+ if (l.ram.empty() || ram_free.empty() || ram_inflight >= kMaxSwaps) continue;+ int rb = -1; // RAM: the hottest read from the file, in place of the layer's coldest held (or a free slot)+ for (int e : hits)+ if (e != vin && l.slot[(size_t) e] < 0 && l.ram[(size_t) e] < 0 &&+ (rb < 0 || l.heat[(size_t) e] > l.heat[(size_t) rb]))+ rb = e;+ if (rb < 0) continue;+ int rc = -1;+ for (int e = 0; e < l.n_exp; ++e)+ if (l.ram[(size_t) e] >= 0 && (rc < 0 || l.heat[(size_t) e] < l.heat[(size_t) rc])) rc = e;+ const bool room = ram_free.size() > 1; // (slots VRAM freed: filled at once)+ if (!room && !(rc >= 0 && l.heat[(size_t) rb] > l.heat[(size_t) rc] * 1.25f + 1.0f)) continue;+ RamSwap rs;+ rs.layer = l.il;+ rs.in = rb;+ rs.out = room ? -1 : rc;+ rs.slot = ram_free.back();+ ram_free.pop_back();+ ++ram_inflight;+ std::lock_guard<std::mutex> lk(sw_mu);+ ram_queue.push_back(rs);+ rqueued = true;+ }+ if (queued) sw_cv.notify_all();+ if (rqueued) ram_cv.notify_one();+ }++ // lookahead's thread: the ranges the CPU lane asked for, read in (os::prefetch) off the lane, which computes+ void prefetch_loop() {+ std::vector<std::pair<const void*, size_t>> b;+ for (;;) {+ {+ std::unique_lock<std::mutex> lk(pf_mu);+ pf_cv.wait(lk, [&] { return quit.load(std::memory_order_relaxed) || !pf_q.empty(); });+ if (quit.load(std::memory_order_relaxed)) return;+ b.swap(pf_q);+ }+ os::prefetch_cold(b); // (the resident ones would only cost the page walk)+ b.clear();+ }+ }+ // an expert the CPU lane reads from the files (neither in VRAM nor in the RAM tier)+ static bool file_read(const Layer& l, int e) {+ return l.slot[(size_t) e] < 0 && (l.ram.empty() || l.ram[(size_t) e] < 0);+ }+ // the layer whose picks layer l0 guesses a layers ahead (1, 2): the next ones of its stack on its device, MXFP4+ // experts read in place, some of them from the files now (none: nothing to read early - null)+ const Layer* ahead_layer(const Layer& l0, int a) const {+ const int j = l0.il + a, end = l0.mtp ? g.nl + g.n_mtp : g.nl;+ if (j >= end || j >= (int) L.size()) return nullptr;+ const Layer& n = L[(size_t) j];+ if (n.dev != l0.dev || n.mtp != l0.mtp || n.lay.fmt != gg::kFmtMXFP4 || n.gate_w == nullptr || n.n_exp > 1024 ||+ (int) n.slot.size() < n.n_exp)+ return nullptr;+ for (int e = 0; e < n.n_exp; ++e)+ if (file_read(n, e)) return &n;+ return nullptr;+ }+ // lookahead on the CPU lane: request r (layer li, seq) carries the device's guesses for the next layers - the+ // guessed experts read from the files asked for now (each once a window), the guesses kept for the hit counts+ std::vector<std::pair<const void*, size_t>> pf_want;+ void ahead_ask(int li, const Req& r, unsigned seq, int T) {+ pf_want.clear();+ for (int a = 1; a <= 2; ++a) {+ const int m = std::min(r.na[a - 1], a == 1 ? kAhead1 : kPicks), j = li + a;+ if (m <= 0 || j >= (int) L.size()) continue;+ const Layer& n = L[(size_t) j];+ uint8_t* fl = ahead_fl.data() + (size_t) j * 1024;+ uint8_t* rk = ahead_rank.data() + (size_t) j * 1024;+ if (ahead_for[(size_t) j] != seq + (unsigned) a) {+ std::memset(fl, 0, 1024);+ std::memset(rk, 0, 1024);+ ahead_for[(size_t) j] = seq + (unsigned) a;+ }+ for (int t = 0; t < T; ++t)+ for (int i = 0; i < m; ++i) {+ const int e = r.ahead[t][(a - 1) * kAhead1 + i];+ if (e < 0 || e >= n.n_exp || e >= (int) n.slot.size()) continue;+ fl[e] |= a == 2 ? 4 : (i < n.k_exp ? 3 : 2);+ if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);+ if ((fl[e] & 8) || !file_read(n, e)) continue;+ fl[e] |= 8;+ if (store) store->prefetch(j, e);+ else+ for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});+ ++st.ahead_pf;+ }+ }+ if (pf_want.empty()) return;+ {+ std::lock_guard<std::mutex> lk(pf_mu);+ pf_q.insert(pf_q.end(), pf_want.begin(), pf_want.end());+ }+ pf_cv.notify_one();+ }++ // the decode path's host side: each published routing in turn - its CPU lane experts (if any) into the answer's+ // part, then the answer's seq. Spins while requests come (a token's 40 layers), naps after 2 ms without one.+ void serve_lane() {+ unsigned next = 1;+ auto last = std::chrono::steady_clock::now();+ std::vector<Ent> ents;+ while (!quit.load(std::memory_order_relaxed)) {+ Req& r = reqs[next % kRing];+ if (*(volatile unsigned*) &r.seq != next) {+ cpu_relax();+ if (std::chrono::steady_clock::now() - last > std::chrono::milliseconds(2))+ std::this_thread::sleep_for(std::chrono::microseconds(20));+ continue;+ }+ std::atomic_thread_fence(std::memory_order_acquire);+ const int li = layer_of[next % kRing].load(std::memory_order_acquire);+ Resp& rp = resps[next % kRing];+ const int T = std::max(1, std::min(kWin, r.T));+ for (int t = 0; t < T; ++t) rp.cpu[t] = 0;+ if (li >= 0 && li < (int) L.size()) {+ Layer& l = L[(size_t) li];+ uint32_t* pr = prof.data() + (size_t) li * g.n_exp;+ ents.clear();+ if (store && li > 0 && li < g.nl) store->cancel_stale(li - 1); // (guesses for layers the token has passed: dropped unread)+ // lookahead: the guesses made for this layer (when they are for this request), counted on decode+ const uint8_t* fl = T == 1 && !ahead_for.empty() && ahead_for[(size_t) li] == next+ ? ahead_fl.data() + (size_t) li * 1024 : nullptr;+ // the window's CPU-lane picks by expert (up to 8 tokens an entry), their weights by entry row+ float wv[kWin * kPicks];+ int tok_of[kWin * kPicks];+ int nslots = 0;+ std::vector<std::pair<int, int>> picks; // (expert, t * kPicks + i), CPU-lane only+ if (trace_f != nullptr && !l.mtp) { // DS4_TRACE=file: one record a route request: u8 layer, u8 T, T x k int16 expert ids+ // DS4_TRACE_AHEAD=1: T has bit 7 set and the guesses follow: u8 na1, u8 na2, T x (na1 + na2) int16 (the+ // next layer's likely picks by rank, then the one after's), as the lookahead made them for THIS layer's+ // predecessors (the request carries the guesses for the layers after it)+ static const bool with_ahead = std::getenv("DS4_TRACE_AHEAD") != nullptr;+ uint8_t hd[2] = {(uint8_t) li, (uint8_t) (T | (with_ahead ? 0x80 : 0))};+ std::fwrite(hd, 1, 2, trace_f);+ for (int t = 0; t < T; ++t)+ for (int i = 0; i < l.k_exp; ++i) {+ const int16_t v = (int16_t) r.ids[t][i];+ std::fwrite(&v, 2, 1, trace_f);+ }+ if (with_ahead) {+ const uint8_t na[2] = {(uint8_t) std::max(0, std::min(r.na[0], kAhead1)), (uint8_t) std::max(0, std::min(r.na[1], kPicks))};+ std::fwrite(na, 1, 2, trace_f);+ for (int t = 0; t < T; ++t) {+ for (int i = 0; i < na[0]; ++i) {+ const int16_t v = (int16_t) r.ahead[t][i];+ std::fwrite(&v, 2, 1, trace_f);+ }+ for (int i = 0; i < na[1]; ++i) {+ const int16_t v = (int16_t) r.ahead[t][kAhead1 + i];+ std::fwrite(&v, 2, 1, trace_f);+ }+ }+ }+ }+ const bool dsk = disk_skip_w > 0.0f && !l.mtp && li >= disk_skip_from && li < g.nl;+ for (int t = 0; t < T; ++t) {+ float wsum = 0.0f; // (DISK_SKIP_W: the token's route weights)+ if (dsk)+ for (int i = 0; i < l.k_exp; ++i)+ if (r.ids[t][i] >= 0 && r.ids[t][i] < l.n_exp) wsum += r.w[t][i];+ for (int i = 0; i < l.k_exp; ++i) {+ const int e = r.ids[t][i];+ if (e < 0 || e >= l.n_exp) continue;+ ++pr[e];+ l.heat[(size_t) e] += 1.0f;+ if (fl != nullptr) {+ ++st.ahead_picks;+ st.ahead_hit_k += fl[e] & 1;+ st.ahead_hit += (fl[e] & 6) != 0;+ st.ahead_hit2 += (fl[e] >> 2) & 1;+ }+ if (l.slot[(size_t) e] < 0) {+ l.cpu_hits.push_back(e);+ if (T == 1 && !l.zc.empty() && l.zc[(size_t) e]) { // the device computed it from the store's slot+ store->touch(li, e);+ ++st.store_zc;+ continue;+ }+ if (dsk && r.w[t][i] < disk_skip_w * wsum && !in_ram(l, e)) {+ ++st.skipped;+ continue;+ }+ if (!l.ram.empty() && l.ram[(size_t) e] >= 0) ++st.ram_experts;+ else if (fl != nullptr) {+ ++st.ahead_cold;+ st.ahead_cold_hit += (fl[e] >> 3) & 1;+ }+ picks.push_back({e, t * kPicks + i});+ }+ }+ }+ if (fl != nullptr) { // the guesses by rank: how many were picked+ const uint8_t* rk = ahead_rank.data() + (size_t) li * 1024;+ for (int e = 0; e < l.n_exp && e < 1024; ++e) {+ if (rk[e] == 0 || rk[e] > 16) continue;+ bool hit = false;+ for (int i = 0; i < l.k_exp; ++i) hit = hit || r.ids[0][i] == e;+ ++st.ahead_rank_n[rk[e] - 1];+ st.ahead_rank_hit[rk[e] - 1] += hit;+ }+ }+ if (ahead_mode > 0 && !ahead_for.empty()) ahead_ask(li, r, next, T);+ std::sort(picks.begin(), picks.end());+ for (size_t a = 0; a < picks.size();) {+ Ent en;+ en.e = picks[a].first;+ en.slot0 = nslots;+ en.n = 0;+ while (a < picks.size() && picks[a].first == en.e && en.n < ent_cap(l)) {+ const int t = picks[a].second / kPicks, i = picks[a].second % kPicks;+ en.tok[en.n++] = t;+ tok_of[nslots] = t;+ wv[nslots] = r.w[t][i];+ ++nslots;+ ++a;+ }+ ents.push_back(en);+ }+ // lookahead: the entries asked for early (or in the RAM tier) computed first, so the ones asked for+ // only now get the others' time to come in from the SSD (their slots, so the sums, stay the same)+ int ord[kWin * kPicks];+ const int* po = nullptr;+ if (ahead_order && !ahead_for.empty() && ahead_for[(size_t) li] == next && ents.size() > 1 &&+ ents.size() <= (size_t) (kWin * kPicks)) {+ const uint8_t* fa = ahead_fl.data() + (size_t) li * 1024;+ int n = 0;+ for (int pass = 0; pass < 2; ++pass)+ for (size_t q = 0; q < ents.size(); ++q) {+ const int e = ents[q].e;+ const bool early = !file_read(l, e) || (fa[e] & 8) != 0;+ if (early == (pass == 0)) ord[n++] = (int) q;+ }+ po = ord;+ }+ if (!ents.empty()) {+ const double tc = now_ms();+ cpu_experts(l, &r.x[0][0], ents, (size_t) kMaxDim, po);+ const int D = g.dim;+ for (int t = 0; t < T; ++t) {+ bool any = false;+ for (int sl = 0; sl < nslots; ++sl) any = any || tok_of[sl] == t;+ if (!any) continue;+ float* o = rp.part[t];+ for (int e = 0; e < D; ++e) o[e] = 0.0f;+ for (int sl = 0; sl < nslots; ++sl) {+ if (tok_of[sl] != t) continue;+ const float* y = c_yd.data() + (size_t) sl * D;+ for (int e = 0; e < D; ++e) o[e] += wv[sl] * y[e];+ }+ rp.cpu[t] = 1;+ }+ st.cpu_ms += now_ms() - tc;+ st.cpu_experts += nslots;+ }+ }+ std::atomic_thread_fence(std::memory_order_release);+ *(volatile unsigned*) &rp.seq = next;+ ++next;+ last = std::chrono::steady_clock::now();+ }+ }++ // a GGUF's matrices (BF16 / MXFP4) to its own kernels, EXL3 ones to x3's+ static bool mv_any(const x3::MvJob* j, int n, cudaStream_t s) {+ bool g = false;+ for (int i = 0; i < n; ++i) g = g || j[i].m->fmt != 0;+ return g ? gg::mv(j, n, s) : x3::mv(j, n, s);+ }+ bool mv(Dev& d, std::initializer_list<x3::MvJob> jobs, std::string& err) {+ x3::MvJob j[x3::kMaxMvJobs];+ int n = 0;+ for (const x3::MvJob& x : jobs) j[n++] = x;+ if (!mv_any(j, n, d.st)) {+ err = "exl3 mv failed";+ return false;+ }+ return true;+ }+ bool mv_n(Dev& d, x3::MvJob* j, int n, std::string& err) {+ for (int a = 0; a < n; a += x3::kMaxMvJobs)+ if (!mv_any(j + a, std::min(x3::kMaxMvJobs, n - a), d.st)) {+ err = "exl3 mv failed";+ return false;+ }+ return true;+ }+ // y[c] = x[c] W for the T activations x + c * x_cs (floats), into y + c * y_cs+ static x3::MvJob job(const x3::Mat& m, const float* x, float* y, int T = 1, size_t x_cs = 0, size_t y_cs = 0,+ float alpha = 1.0f) {+ x3::MvJob j;+ j.m = &m;+ j.xf = x;+ j.y = y;+ j.alpha = alpha;+ j.ncol = T;+ j.xf_cs = x_cs;+ j.y_cs = y_cs;+ return j;+ }++ // ---- the DSpark drafter (DeepSeek's DSparkBlock / forward_spec). Its blocks' rings hold the trunk's main KV:+ // seed() turns a window's taps (the streams' mean entering layers 37..39) into it, through main_proj / main_norm+ // and each block's wkv / kv_norm / rope / FP8, for the positions the trunk kept. draft() runs the block [y, noise x+ // (block - 1)] at positions pos .. through the blocks (each query attending to the ring and to the whole block),+ // the trunk's head, the markov bias token by token (greedy) and the confidence head.+ bool seed_drafter(int64_t p0, int n, int row0, std::string& err) {+ if (g.n_mtp == 0 || n <= 0) return true;+ if (n > kWin * kPicks) {+ err = "seeding the drafter with " + std::to_string(n) + " rows";+ return false;+ }+ Dev& d = devs[(size_t) d_dev];+ cudaSetDevice(d.id);+ const int nt = (int) g.d_targets.size(), D = g.dim, HD = g.hd;+ if (dump_d != nullptr) { // DS4_DSPARK_DUMP: 'S', the first position, the rows, then the taps+ for (Dev& e : devs) {+ cudaSetDevice(e.id);+ CK(cudaStreamSynchronize(e.st));+ }+ cudaSetDevice(d.id);+ const int64_t hdr[4] = {'S', p0, n, (int64_t) nt * D};+ std::fwrite(hdr, 8, 4, dump_d);+ std::fwrite(h_taps + (size_t) row0 * nt * D, 4, (size_t) n * nt * D, dump_d);+ std::fflush(dump_d);+ }+ for (int a = 0; a < n; a += kWin) { // (a chunk's last 128 rows: 16 a GEMV)+ const int na = std::min(kWin, n - a);+ CK(cudaMemcpyAsync(d.xe, h_taps + (size_t) (row0 + a) * nt * D, (size_t) na * nt * D * 4,+ cudaMemcpyHostToDevice, d.st));+ if (!mv(d, {job(d_main_proj, d.xe, d.ed + (size_t) a * D, na, (size_t) nt * D, D)}, err)) return false;+ }+ CK(cudaEventRecord(ev_seed, d.st));+ Finish fm;+ fm.w = d_main_norm;+ fm.eps = g.eps;+ finish(d.ed, n, D, D, fm, d.st);+ for (int i = 0; i < g.n_mtp; ++i) {+ Layer& l = L[(size_t) (g.nl + i)];+ if (l.dev != d_dev) {+ err = "the DSpark blocks are split across devices";+ return false;+ }+ for (int a = 0; a < n; a += kWin) {+ const int na = std::min(kWin, n - a);+ if (!mv(d, {job(l.wkv, d.ed + (size_t) a * D, d.kvr, na, D, HD)}, err)) return false;+ Finish fk;+ fk.w = l.kv_norm;+ fk.eps = g.eps;+ fk.rd = g.rd;+ fk.pos = (int) (p0 + a);+ fk.freqs = d.freqs_win;+ fk.quant = 1;+ fk.dst = d.ext + (size_t) g.win * HD;+ fk.dst_stride = HD;+ finish(d.kvr, na, HD, HD, fk, d.st);+ ext_to_ring(d.ext, l.win, p0 + a, g.win, na, HD, d.st);+ }+ }+ return true;+ }++ bool draft(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err, float temperature, float top_p,+ int top_k, float min_p, uint64_t seed, uint64_t counter) {+ const double t0 = now_ms();+ drafts_q = temperature > 0.0f && sample_drafts;+ dq_temp = temperature;+ dq_top_p = top_p;+ dq_top_k = top_k;+ dq_min_p = min_p;+ dq_seed = seed;+ dq_counter = counter;+ const bool ok = draft_(y, out, conf, err);+ if (!ok) drafts_q = false;+ if (max_drafts > 0 && (int) out.size() > max_drafts) {+ out.resize((size_t) max_drafts);+ conf.resize((size_t) max_drafts);+ }+ st.draft_ms += now_ms() - t0;+ ++st.drafts_rounds;+ return ok;+ }+ bool draft_(int y, std::vector<int>& out, std::vector<float>& conf, std::string& err) {+ out.clear();+ conf.clear();+ const int T = g.d_block, D = g.dim, H = g.hc, V = g.vocab, R = g.d_rank;+ const int64_t p0 = pos;+ if (g.n_mtp == 0 || T < 1 || T > kWin || p0 + T > o.max_context) return true;+ int tk[kWin];+ tk[0] = y;+ for (int t = 1; t < T; ++t) tk[t] = g.d_noise;+ Dev& d = devs[(size_t) d_dev];+ cudaSetDevice(d.id);+ for (int t = 0; t < T; ++t) std::memcpy(h_emb + (size_t) t * D, embed_pinned + (size_t) tk[t] * D, (size_t) D * 2);- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));++ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));+ embed(d.emb_rows, T, d.h, D, H, d.st);++ if (std::getenv("DS4_DBG_EMB")) {++ ds4_metal_flush();++ float *hp = d.h;++ float hh[4] = {0,0,0,0};++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);++ }++ if (std::getenv("DS4_DBG_EMB")) {++ ds4_metal_flush();++ float hh[4] = {0, 0, 0, 0};++ cudaMemcpy(hh, d.h, sizeof(hh), cudaMemcpyDeviceToHost);++ std::fprintf(stderr, "DBG after embed: h[0..3] = %g %g %g %g\n", hh[0], hh[1], hh[2], hh[3]);++ }+ init_pre(d.pre, T, d.st);+ WinPlan dp; // (window attention alone: the mailbox seqs)+ dp.async = !sync_decode && T <= async_max_t;+ for (int i = 0; i < g.n_mtp && dp.async; ++i) {+ const int il = g.nl + i;+ dp.seq[il] = ++seq_issued;+ layer_of[dp.seq[il] % kRing].store(il, std::memory_order_release);+ }+ const size_t hs = (size_t) T * H * D, rs = (size_t) g.win * g.hd;+ if (dump_d != nullptr && h_dump == nullptr &&+ (!halloc(h_dump, (size_t) g.n_mtp * kWin * H * D, err) || !halloc(h_dump_ring, (size_t) g.n_mtp * rs, err)))+ return false;+ for (int i = 0; i < g.n_mtp; ++i) {+ Layer& l = L[(size_t) (g.nl + i)];+ if (dump_d != nullptr) { // (stream order: the ring the block reads - as BF16 rows - the stream it leaves)+ ring_to_ext(l.win, d.ext, g.win, g.win, g.hd, d.st);+ CK(cudaMemcpyAsync(h_dump_ring + (size_t) i * rs, d.ext, rs * 2, cudaMemcpyDeviceToHost, d.st));+ }+ if (!layer(l, 0, p0, T, dp, err)) return false;+ if (dump_d != nullptr) {+ cudaSetDevice(d.id);+ CK(cudaMemcpyAsync(h_dump + (size_t) i * hs, d.h, hs * 4, cudaMemcpyDeviceToHost, d.st));+ }+ }+ cudaSetDevice(d.id);+ hc_pre_norm(d.h, d.pre, nullptr, D, H, T, g.eps, d.x, d.st); // (the confidence head reads it before the norm)+ Dev& hd = devs[(size_t) head_dev];+ // (through the host to the head's device, and to the head group's other members - the drafter's device can be+ // the head group's lead itself, a group of every GPU)+ if (d_dev != head_dev || groups[(size_t) hd.grp].mem.size() > 1) {+ CK(cudaMemcpyAsync(d.bounce, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, d.st));+ CK(cudaEventRecord(d.ev_out, d.st));+ }+ if (d_dev != head_dev) {+ cudaSetDevice(hd.id);+ CK(cudaStreamWaitEvent(hd.st, d.ev_out, 0));+ CK(cudaMemcpyAsync(hd.x, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, hd.st));+ }+ // the drafter's final norm and the trunk's head, on every member of the head group (the others' shares first)+ const std::vector<int>& hg = groups[(size_t) hd.grp].mem;+ const bool mk = mk_split && !drafts_q; // (sampled drafts: the lead's whole rows, gathered)+ for (int m = (int) hg.size() - 1; m >= 0; --m) {+ Dev& e = devs[(size_t) hg[(size_t) m]];+ cudaSetDevice(e.id);+ if (m > 0) {+ CK(cudaStreamWaitEvent(e.st, d.ev_out, 0));+ CK(cudaMemcpyAsync(e.ao, d.bounce, (size_t) T * D * 4, cudaMemcpyHostToDevice, e.st));+ } else+ CK(cudaMemcpyAsync(e.ao, e.x, (size_t) T * D * 4, cudaMemcpyDeviceToDevice, e.st));+ Finish fn;+ fn.w = e.d_norm;+ fn.eps = g.eps;+ finish(e.ao, T, D, D, fn, e.st);+ if (!head_share(e, m, e.ao, T, err, !mk)) return false;+ }+ h_idx[0] = y;+ if (mk) {+ // the markov steps over the group: each member its rows of the bias and its share's argmax, every member+ // then the global one (the sampler's tie rule) and its E row - the same drafts as the lead's whole rows,+ // each step ~1/nm of their 66 MB and no gather of the shares+ const int nm = (int) hg.size();+ const unsigned seq = ++mk_seq;+ for (int m = 0; m < nm; ++m) {+ Dev& e = devs[(size_t) hg[(size_t) m]];+ cudaSetDevice(e.id);+ CK(cudaMemcpyAsync(e.mk_ints, h_idx, 4, cudaMemcpyHostToDevice, e.st));+ CK(cudaMemcpyAsync(e.mk_erow, h_mk_embed + (size_t) y * R, (size_t) R * 2, cudaMemcpyHostToDevice, e.st));+ }+ for (int i = 0; i < T; ++i)+ for (int m = 0; m < nm; ++m) {+ Dev& e = devs[(size_t) hg[(size_t) m]];+ cudaSetDevice(e.id);+ float* row = m == 0 ? hd.logits + (size_t) i * V : e.logits_part + (size_t) i * e.mk_vn;+ markov_share(row, e.mk_share, e.mk_erow, e.mk_vn, R, e.st);+ argmax_share(row, e.mk_vn, e.mk_v0, mk_slots + ((size_t) i * nm + m) * 4, seq, e.st);+ markov_pick(mk_slots + (size_t) i * nm * 4, nm, seq, e.mk_ints + i + 1, h_mk_embed, e.mk_erow, R,+ e.st);+ }+ cudaSetDevice(hd.id);+ } else {+ CK(cudaMemcpyAsync(d_ints, h_idx, 4, cudaMemcpyHostToDevice, hd.st));+ strata::dsv41k::SamplerParams sp;+ sp.greedy = true;+ sp.temperature = 0.0f;+ if (drafts_q) { // the request's own chain: draft i drawn as the model's token i would be (verify's counter + i)+ sp.greedy = false;+ sp.temperature = dq_temp * (q_temp_scale > 0.0f ? q_temp_scale : 1.0f);+ sp.top_p = dq_top_p;+ sp.top_k = dq_top_k;+ sp.min_p = dq_min_p;+ sp.seed = dq_seed;+ }+ for (int i = 0; i < T; ++i) { // the markov bias of the token before, then that position's draft+ if (d_mk_head_q8 != nullptr) markov_add_q8(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head_q8, d_ints + i, V, R, hd.st);+ else markov_add(hd.logits + (size_t) i * V, d_mk_embed, d_mk_head, d_ints + i, V, R, hd.st);+ if (drafts_q) {+ sp.counter = dq_counter + (uint64_t) i;+ strata::dsv41k::sample_draft(hd.logits + (size_t) i * V, 1, V, sp, d_specq + i, d_ints + i + 1, hd.st);+ } else+ strata::dsv41k::sample_tokens(hd.logits + (size_t) i * V, 1, V, nullptr, 0, sp, d_ints + i + 1, hd.st);+ }+ }+ confidence(hd.x, d_mk_embed, d_ints, d_conf, T, D, R, d_confv, hd.st);+ int tok[kWin + 1];+ float cf[kWin];++#if defined(STRATA_METAL)++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)++#endif+ CK(cudaMemcpyAsync(h_idx, d_ints, (size_t) (T + 1) * 4, cudaMemcpyDeviceToHost, hd.st));+ CK(cudaMemcpyAsync(h_w, d_confv, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));+ CK(cudaStreamSynchronize(hd.st));+ for (int i = 0; i <= T; ++i) tok[i] = h_idx[i];+ for (int i = 0; i < T; ++i) cf[i] = h_w[i];+ if (dump_d != nullptr) { // 'D', the block's position, y, T, then the drafts, their confidences, the hidden+ std::vector<float> xs((size_t) T * D); // rows before the final norm; then each block's ring and stream+ CK(cudaMemcpy(xs.data(), hd.x, xs.size() * 4, cudaMemcpyDeviceToHost));+ cudaSetDevice(d.id);+ CK(cudaStreamSynchronize(d.st));+ const int64_t hdr[4] = {'D', p0, y, T};+ std::fwrite(hdr, 8, 4, dump_d);+ std::fwrite(tok + 1, 4, (size_t) T, dump_d);+ std::fwrite(cf, 4, (size_t) T, dump_d);+ std::fwrite(xs.data(), 4, xs.size(), dump_d);+ for (int i = 0; i < g.n_mtp; ++i) {+ std::fwrite(h_dump_ring + (size_t) i * rs, 2, rs, dump_d);+ std::fwrite(h_dump + (size_t) i * hs, 4, hs, dump_d);+ }+ std::fflush(dump_d);+ cudaSetDevice(hd.id);+ }+ for (int i = 0; i < T; ++i) {+ out.push_back(tok[i + 1]);+ conf.push_back(cf[i]);+ }+ return true;+ }++ int sample_row(int row, float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter,+ const float* bias) {+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ strata::dsv41k::SamplerParams sp;+ sp.temperature = temperature;+ sp.greedy = temperature <= 0.0f;+ sp.top_p = top_p;+ sp.top_k = top_k;+ sp.min_p = min_p;+ sp.seed = seed;+ sp.counter = counter;+ sp.logit_bias = bias;+ strata::dsv41k::sample_tokens(d.logits + (size_t) row * g.vocab, 1, g.vocab, nullptr, 0, sp, d.tok, d.st);+ int t = -1;++#if defined(STRATA_METAL)++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)++#endif+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);+ cudaStreamSynchronize(d.st);+ return t;+ }++ // [y, drafts] at positions pos ..: the trunk's verdict, its next token, and the state back to what it kept.+ // Greedy: the drafts the trunk's argmax agrees with. Sampled: the speculative-sampling rule over the request's own+ // chain (top_k, top_p, min_p, temperature): draft d stands with probability min(1, p(d) / q(d)), q the distribution+ // it was drawn from (the drafter's argmax: a point at d, so p(d)), a rejected one is replaced by a draw from the+ // residual max(0, p - q) - so each kept token is distributed as the trunk's own draw.+ bool verify(int y, const std::vector<int>& drafts, float temperature, float top_p, int top_k, float min_p,+ uint64_t seed, uint64_t counter, int& accepted, int& next, std::string& err) {+ const int n = (int) std::min<size_t>(drafts.size(), (size_t) kWin - 1);+ const int T = n + 1, V = g.vocab;+ const int64_t p0 = pos;+ int tk[kWin];+ tk[0] = y;+ for (int i = 0; i < n; ++i) tk[i + 1] = drafts[(size_t) i];+ spec_win = true;+ const double tv = now_ms();+ const bool ok = forward(tk, T, T, err, 0);+ st.verify_ms += now_ms() - tv;+ spec_win = false;+ if (!ok) return false;+ Dev& hd = devs[(size_t) head_dev];+ cudaSetDevice(hd.id);+ int m = 0;+ if (temperature <= 0.0f) {+ strata::dsv41k::SamplerParams sp;+ sp.greedy = true;+ sp.temperature = 0.0f;+ strata::dsv41k::sample_tokens(hd.logits, T, V, nullptr, 0, sp, d_ints + 16, hd.st); // (the T rows at once)++#if defined(STRATA_METAL)++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)++#endif+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) T * 4, cudaMemcpyDeviceToHost, hd.st));+ CK(cudaStreamSynchronize(hd.st));+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;+ next = h_idx[m];+ } else {+ // row i keeps draft i with probability min(1, p(d) / q(d)), else draws from the residual max(0, p - q)+ // (never d itself): each row a token distributed as the model's own draw at that position, p through the+ // request's chain - the top_k / top_p / min_p cut included (the drafter's argmax can lie outside it: p(d) 0,+ // never kept); past the first rejection the rows do not count+ const bool own = drafts_q && dq_temp == temperature && dq_top_p == top_p && dq_top_k == top_k &&+ dq_min_p == min_p && dq_seed == seed && dq_counter == counter;+ if (!own) { // the drafter's argmax: q a point at each draft+ for (int i = 0; i < n; ++i) {+ h_specq[(size_t) i].n = 1;+ h_specq[(size_t) i].ids[0] = drafts[(size_t) i];+ h_specq[(size_t) i].p[0] = 1.0;+ }+ CK(cudaMemcpyAsync(d_specq, h_specq.data(), (size_t) n * sizeof(strata::dsv41k::SpecList),+ cudaMemcpyHostToDevice, hd.st));+ }+ for (int i = 0; i < n; ++i) h_idx[i] = drafts[(size_t) i];+ CK(cudaMemcpyAsync(d_ints + 32, h_idx, (size_t) n * 4, cudaMemcpyHostToDevice, hd.st));+ strata::dsv41k::SamplerParams sp;+ sp.greedy = false;+ sp.temperature = temperature;+ sp.top_p = top_p;+ sp.top_k = top_k;+ sp.min_p = min_p;+ sp.seed = seed;+ sp.counter = counter;+ strata::dsv41k::sample_verify(hd.logits, n, V, sp, d_specq, d_ints + 32, d_ints + 16, hd.st);++#if defined(STRATA_METAL)++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)++#endif+ CK(cudaMemcpyAsync(h_idx, d_ints + 16, (size_t) n * 4, cudaMemcpyDeviceToHost, hd.st));+ CK(cudaStreamSynchronize(hd.st));+ while (m < n && h_idx[m] == drafts[(size_t) m]) ++m;+ next = m < n ? h_idx[m] : sample_row(n, temperature, top_p, top_k, min_p, seed, counter + (uint64_t) n, nullptr);+ }+ drafts_q = false;+ rollback(p0, T, m + 1);+ if (!seed_drafter(p0, m + 1, 0, err)) return false;+ accepted = m;+ st.drafts_offered += n;+ st.drafts_accepted += m;+ for (int i = 0; i < n && i <= m && i < 8; ++i) {+ ++st.draft_reached[i];+ if (i < m) ++st.draft_kept[i];+ }+ return true;+ }++ // a verified window keeps its first `keep` positions: the ring rows and the compressors' state of the rest undone+ // (the compressed KV and the index keys past them are written again when those positions come)+ void rollback(int64_t p0, int T, int keep) {+ if (keep >= T) return;+ // the ring rows: one launch a device for all its layers (a launch a layer and member was ~300 on 8-way tensor+ // parallel, a millisecond of the host's before the next draft)+ std::vector<RingBatch> rb(devs.size());+ for (int il = 0; il < g.nl; ++il)+ for (int m = 0; m < L[(size_t) il].nm; ++m) {+ Layer& l = member(L[(size_t) il], m);+ RingBatch& b = rb[(size_t) l.dev];+ if (b.n == kMaxRingBatch) { // (more layers on a device than a batch takes: this one alone)+ cudaSetDevice(devs[(size_t) l.dev].id);+ ring_restore(l.win, l.ring_bak, p0, g.win, keep, T, g.hd, devs[(size_t) l.dev].st);+ continue;+ }+ b.ring[b.n] = l.win;+ b.bak[b.n++] = l.ring_bak;+ }+ for (size_t di = 0; di < devs.size(); ++di)+ if (rb[di].n > 0) {+ cudaSetDevice(devs[di].id);+ ring_restore_batch(rb[di], p0, g.win, keep, T, g.hd, devs[di].st);+ }+ for (int il = 0; il < g.nl; ++il)+ for (int m = 0; m < L[(size_t) il].nm; ++m) {+ Layer& l = member(L[(size_t) il], m);+ Dev& d = devs[(size_t) l.dev];+ if (l.c_snap != nullptr) {+ cudaSetDevice(d.id);+ const size_t ns = (size_t) l.ratio * g.hd * 4;+ cudaMemcpyAsync(l.c_kv, l.c_snap, ns, cudaMemcpyDeviceToDevice, d.st);+ cudaMemcpyAsync(l.c_score, l.c_snap + (size_t) l.ratio * g.hd, ns, cudaMemcpyDeviceToDevice, d.st);+ compress_win(l.c_rows, l.c_rows + (size_t) kWin * g.hd, l.c_kv, l.c_score, l.ratio, (int) p0, keep,+ g.hd, l.c_norm, g.eps, d.latent, d.st);+ }+ }+ pos = p0 + keep;+ toks.resize((size_t) pos);+ ctoks.resize((size_t) pos);+ slot_held = false;+ }++ // ---- a window: T tokens at positions pos .. pos + T - 1 through every layer; the logits of its last `logits` rows+ bool step(int token, bool want_logits, std::string& err) { return forward(&token, 1, want_logits ? 1 : 0, err); }+++ // layer L0 for a chunk of T tokens at p0 as group member mi: layer()'s arithmetic over the chunk's rows (the+ // projections as GEMMs, the attention and the indexer in slabs, the routed experts as sorted rows: the VRAM ones in+ // place, the CPU lane's streamed in batches). kv_only (a long prompt's first pass, Run::stop): only what the layers+ // after it read of these rows - its compressed KV and index keys - and nothing of its own attention or experts+ bool layer_chunk(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err,+ bool kv_only = false) {+ Layer& l = member(L0, mi);+ Dev& d = devs[(size_t) l.dev];+ Chunk& c = *chunks[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ prof_mark(d, 0);+ const int nm = L0.nm;+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;+ const ArGroup& ab = groups[(size_t) L0.grp].arb;+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;+ // Engram, in slabs of 256 tokens+ if (l.eg >= 0) {+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;+ const size_t per = (size_t) cols * (ehd + ehd / 32);+ const int slot = wp.eg_slot >= 0 ? wp.eg_slot : wp.k % 2; // (read ahead by the run: two in flight)+ uint8_t* host = c.eg_host[slot] + (size_t) l.eg * c.C * per;+ CK(cudaMemcpyAsync(c.eg_rows, host, (size_t) T * per, cudaMemcpyHostToDevice, s));+ CK(cudaEventRecord(c.eg_ev[slot], s));+ for (int s0 = 0; s0 < T; s0 += 256) {+ const int ns = std::min(256, T - s0);+ engram_dequant_tok(c.eg_rows + (size_t) s0 * per, per, cols, ns, ehd, c.eg_x, s, (int) eg_fmt[(size_t) l.eg]);+ x3::rows_f16(c.eg_x, cols * ehd, nullptr, ns, cols * ehd, c.eg_x16, s);+ if (!gemm(d, c, l.eg_wkv, c.eg_x16, cols * ehd, ns, c.eg_kv, (H + 1) * D, 0.0f, err)) return false;+ engram_gate(c.h + (size_t) s0 * H * D, c.eg_kv, l.eg_w, H, D, ns, g.eps, s);+ }+ }+ prof_mark(d, 4);+ // attention- if (!hc_mix_chunk(d, c, c.h, l.hca_fn, T, c.mix, err)) return false;- hc_split(c.mix, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps, T, c.a_pre, c.a_post, c.a_comb, s);++ // the mix and the split in one dispatch (hc_mix + hc_split were two, back to back on the same token)++ hc_mix_split(c.h, l.hca_fn, g.hc * g.dim, (2 + g.hc) * g.hc, T, g.eps, c.mix, l.hca_scale, l.hca_base, H,++ g.sinkhorn, g.hc_eps, c.a_pre, c.a_post, c.a_comb, s);++ dump_sub(d, "mix0", l.il, p0, T, c.mix, (size_t) T * 32 * 4);+ hc_pre_norm(c.h, c.pre, l.attn_norm, D, H, T, g.eps, c.x, s);++ dump_sub(d, "a_mix", l.il, p0, T, c.mix, (size_t) T * 32 * 4);++ dump_sub(d, "a_pre", l.il, p0, T, c.a_pre, (size_t) T * 8 * 4);++ dump_sub(d, "a_post", l.il, p0, T, c.a_post, (size_t) T * 8 * 4);++ dump_sub(d, "a_comb", l.il, p0, T, c.a_comb, (size_t) T * 64 * 4);++ dump_sub(d, "attn_x", l.il, p0, T, c.x, (size_t) T * D * 4);+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);+ prof_mark(d, 5);+ if (!kv_only && (!gemm(d, c, l.wq_a, c.x16, D, T, c.qr, g.q_lora, 0.0f, err) ||+ !gemm(d, c, l.wkv, c.x16, D, T, c.kvr, HD, 0.0f, err)))+ return false;+ if (l.kv_src && !gemm(d, c, l.c_wkv, c.x16, D, T, c.ckr, HD, 0.0f, err)) return false;+ if (l.kv_src && l.ratio > 1 && !gemm(d, c, l.c_wgate, c.x16, D, T, c.cgr, HD, 0.0f, err)) return false;+ prof_mark(d, 6);+ const bool snap = !kv_only && wp.snap_t >= 0 && mi == 0 && L0.snap_win != nullptr;+ if (!kv_only) {++ dump_sub(d, "qr_raw", l.il, p0, T, c.qr, (size_t) T * g.q_lora * 4);++ dump_sub(d, "kvr_raw", l.il, p0, T, c.kvr, (size_t) T * HD * 4);+ Finish fq;+ fq.w = l.q_norm;+ fq.eps = g.eps;+ finish(c.qr, T, g.q_lora, g.q_lora, fq, s);+ x3::rows_f16(c.qr, g.q_lora, nullptr, T, g.q_lora, c.qr16, s);+ // the window rows: the ring's (positions p0 - win ..) then the chunk's+ ring_to_ext(l.win, c.ext, p0, g.win, HD, s);+ Finish fk;+ fk.w = l.kv_norm;+ fk.eps = g.eps;+ fk.rd = g.rd;+ fk.pos = (int) p0;+ fk.freqs = freqs;+ fk.quant = 1;+ fk.dst = c.ext + (size_t) g.win * HD;+ fk.dst_stride = HD;+ finish(c.kvr, T, HD, HD, fk, s);+ // feed_mark's checkpoint: the ring as it was before row snap_t - the old ring, then the chunk's rows before it+ if (snap) {+ CK(cudaMemcpyAsync(L0.snap_win, l.win, (size_t) g.win * win_rb(), cudaMemcpyDeviceToDevice, s));+ if (wp.snap_t > 0) ext_to_ring(c.ext, L0.snap_win, p0, g.win, wp.snap_t, HD, s, nullptr);+ }+ ext_to_ring(c.ext, l.win, p0, g.win, T, HD, s, nullptr);+ }+ const uint8_t* ckv = nullptr;+ const long long ss = (long long) o.max_context + 64;+ if (l.ratio > 0) {+ Src& sr = srcs[(size_t) l.src];+ const int R = l.ratio;+ const int64_t g_first = p0 / R, g_end = (p0 + T) / R;+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);+ const float* latent = c.latent;+ if (l.kv_src) {+ if (R == 1) {+ Finish fc;+ fc.w = l.c_norm;+ fc.eps = g.eps;+ finish(c.ckr, T, HD, HD, fc, s);+ latent = c.ckr;+ } else {+ if (snap && L0.snap_kv != nullptr) { // (each slot: the latest row before snap_t that holds it)+ const int64_t ps = p0 + wp.snap_t;+ for (int sl = 0; sl < R; ++sl) {+ const int64_t q = ps - 1 - (((ps - 1 - sl) % R) + R) % R;+ const float* kv = q >= p0 ? c.ckr + (size_t) (q - p0) * HD : l.c_kv + (size_t) sl * HD;+ const float* sc = q >= p0 ? c.cgr + (size_t) (q - p0) * HD : l.c_score + (size_t) sl * HD;+ CK(cudaMemcpyAsync(L0.snap_kv + (size_t) sl * HD, kv, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));+ CK(cudaMemcpyAsync(L0.snap_sc + (size_t) sl * HD, sc, (size_t) HD * 4, cudaMemcpyDeviceToDevice, s));+ }+ }+ compress_win(c.ckr, c.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, c.latent, s);+ }+ }+ const int clen_max = (int) ((p0 + T) / R);+ if (l.ix_src) {+ Src& ks = srcs[(size_t) l.ik_src];+ if (l.owns_k && ng > 0) {+ x3::rows_f16(latent, HD, nullptr, ng, HD, c.lat16, s);+ if (!gemm(d, c, l.i_wk, c.lat16, HD, ng, c.ikr, g.ix_hd, 0.0f, err)) return false;+ Finish fi;+ fi.w = l.ik_norm;+ fi.eps = g.eps;+ fi.rd = g.rd;+ fi.pos = (int) (g_first * R);+ fi.pos_step = R;+ fi.freqs = freqs;+ fi.quant = 3;+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();+ fi.pdst_stride = (long long) ik_rb();+ finish(c.ikr, ng, g.ix_hd, g.ix_hd, fi, s);+ }+ if (clen_max > 0 && !kv_only) {+ if (!gemm_f16w(c, l.i_wproj, g.ix_heads, D, c.x16, D, T,+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), c.iw, 64, err))+ return false;+ for (int a0 = 0; a0 < T; a0 += c.AI) {+ const int na = std::min(c.AI, T - a0);+ const int64_t pa = p0 + a0;+ if (!gemm(d, c, l.i_wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.iq,+ g.ix_heads * g.ix_hd, 0.0f, err))+ return false;+ Finish fi;+ fi.rd = g.rd;+ fi.pos = (int) pa;+ fi.vpp = g.ix_heads;+ fi.freqs = freqs;+ fi.quant = 3;+ finish(c.iq, na * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);+ Rows lim;+ lim.gen = 1;+ lim.p0 = pa;+ lim.R = R;+ lim.kcap = g.ix_topk;+ lim.ccap = g.cand_topk;+ lim.blk = std::max(1, g.cand_blk);+ const int nlast = (int) ((pa + na) / R); // the slab's widest query+ int* tk = c.topk + (size_t) a0 * topk_stride();+ int* cd = c.cand + (size_t) a0 * cand_stride();+ const float* iw = c.iw + (size_t) a0 * 64;+ if (nlast <= 0) continue;+ if (l.uses_cand && (nlast + g.cand_blk - 1) / g.cand_blk > g.cand_topk) {+ Rows cl = lim;+ cl.gen = 2;+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, g.cand_topk * g.cand_blk,+ cd, (int) cand_stride(), g.cand_blk, cl, c.scores, ss, s);+ Rows tr = lim;+ tr.gen = 3;+ topk(c.scores, ss, na, tr, cd, (int) cand_stride(), g.cand_blk, tk, (int) topk_stride(), s);+ } else {+ index_score(c.iq, iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, na, nlast, nullptr, 0, 1,+ lim, c.scores, ss, s);+ if (l.cand_src) {+ Rows cr = lim;+ cr.kcap = g.cand_topk;+ cand_blocks(c.scores, ss, na, cr, g.cand_blk, c.bm, (long long) bm_stride(), cd,+ (int) cand_stride(), s);+ }+ topk(c.scores, ss, na, lim, nullptr, 0, 1, tk, (int) topk_stride(), s);+ }+ }+ }+ }+ if (l.kv_src && ng > 0) {+ Finish fc;+ fc.rd = g.rd;+ fc.pos = (int) (g_first * R);+ fc.pos_step = R;+ fc.freqs = freqs;+ fc.quant = 2;+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();+ fc.pdst_stride = (long long) ckv_rb();+ finish(const_cast<float*>(latent), ng, HD, HD, fc, s);+ if (mi == 0 && sr.remote) { // the new rows for the other groups' readers (slot k % 2, once+ const int slot = wp.k % 2; // they took the chunk before last's)+ if (wp.k >= 2)+ for (int r : sr.readers) wait_flags(sr.x_ack + ((size_t) slot * devs.size() + r) * 32, 1, 32, wp.win - 2, s);+ Xfer x;+ x.src[0] = fc.pdst;+ x.dst[0] = sr.stage(slot);+ x.bytes[0] = (size_t) ng * ckv_rb();+ x.n = 1;+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();+ x.dst[1] = sr.stage(slot) + x_cap() * ckv_rb();+ x.bytes[1] = (size_t) ng * ik_rb();+ x.n = 2;+ }+ xfer_put(x, sr.flag(slot), wp.win, d.xcnt, s);+ }+ }+ ckv = sr.ckv[(size_t) l.dev];+ }+ if (kv_only) return true;+ prof_mark(d, 7);+ // the attention in slabs: q, the sparse attention, wo_a, wo_b (this member's share) into acc+ if (NHL == 0) CK(cudaMemsetAsync(c.acc, 0, (size_t) T * D * sizeof(*c.acc), s)); // (no heads)+ for (int a0 = 0; a0 < T && NHL > 0; a0 += c.A) {+ const int na = std::min(c.A, T - a0);+ if (!gemm(d, c, l.wq_b, c.qr16 + (size_t) a0 * g.q_lora, g.q_lora, na, c.q, HQ, 0.0f, err)) return false;+ Finish fr;+ fr.rd = g.rd;+ fr.pos = (int) (p0 + a0);+ fr.vpp = NHL;+ fr.freqs = freqs;+ finish(c.q, na * NHL, HD, HD, fr, s);+ prof_mark(d, 8);++ if (a0 == 0 && na == T) dump_sub(d, "q", l.il, p0, T, c.q, (size_t) T * HQ * 4);+ AttnRows ar;+ ar.gen = 1;+ ar.p0 = p0;+ ar.t0 = a0;+ ar.win = g.win;+ ar.R = l.ratio;+ ar.kcap = g.ix_topk;+ sparse_attn(c.q, c.ext, ckv, c.topk + (size_t) a0 * topk_stride(), (int) topk_stride(), ar, l.sink, NHL, HD,+ na, 1.0f / std::sqrt((float) HD), c.o, s, kKvFp4, 0, &d.attn_scr); // (a short last slab)+ prof_mark(d, 9);++ if (a0 == 0 && na == T) dump_sub(d, "o", l.il, p0, T, c.o, (size_t) T * HQ * 4);+ Finish fo;+ fo.rd = g.rd;+ fo.pos = (int) (p0 + a0);+ fo.vpp = NHL;+ fo.freqs = freqs;+ fo.inverse = 1;+ fo.quant = 1;+ finish(c.o, na * NHL, HD, HD, fo, s);++ if (a0 == 0 && na == T) dump_sub(d, "o_fin", l.il, p0, T, c.o, (size_t) T * HQ * 4);+ x3::rows_f16(c.o, HQ, nullptr, na, HQ, c.o16, s);+ const int per = HQ / OG;+ for (int gi = 0; gi < OG; ++gi)+ if (!gemm(d, c, l.wo_a[gi], c.o16 + (size_t) gi * per, HQ, na, c.oa + (size_t) gi * g.o_lora,+ OG * g.o_lora, 0.0f, err))+ return false;+ x3::rows_f16(c.oa, OG * g.o_lora, nullptr, na, OG * g.o_lora, c.oa16, s);+ if (!gemm(d, c, l.wo_b, c.oa16, OG * g.o_lora, na, c.acc + (size_t) a0 * D, D, 0.0f, err)) return false;++ if (a0 == 0 && na == T) dump_sub(d, "ao", l.il, p0, T, c.acc, (size_t) T * D * 4);+ }+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);++ dump_sub(d, "h_in", l.il, p0, T, c.h, (size_t) T * H * D * 4);+ hc_post_inplace(c.acc, c.h, c.a_post, c.a_comb, D, H, T, s);++ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, 1000 + l.il, p0, T); // (after the attention, as the window path)+ prof_mark(d, 1);+ // the MoE- if (!hc_mix_chunk(d, c, c.h, l.hcf_fn, T, c.mix, err)) return false;- hc_split(c.mix, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps, T, c.f_pre, c.f_post, c.f_comb, s);++ // the FFN half of the same fusion: mix and split in one dispatch++ hc_mix_split(c.h, l.hcf_fn, g.hc * g.dim, (2 + g.hc) * g.hc, T, g.eps, c.mix, l.hcf_scale, l.hcf_base, H,++ g.sinkhorn, g.hc_eps, c.f_pre, c.f_post, c.f_comb, s);+ hc_pre_norm(c.h, c.a_pre, l.ffn_norm, D, H, T, g.eps, c.x, s);+ x3::rows_f16(c.x, D, nullptr, T, D, c.x16, s);+ gemv_f32(l.gate_w, c.x, l.n_exp, D, T, c.glog, 1024, s); // (F32 x: the token path's bits)+ for (int r0 = 0; r0 < T; r0 += kVlRows) { // (route()'s image bits cover kVlRows rows a call)+ const int nr = std::min(kVlRows, T - r0);+ VlBits vb;+ const bool vl = l.gate_bias_vl != nullptr && vl_bits(p0 + r0, nr, vb);+ route(c.glog + (size_t) r0 * 1024, l.gate_bias, l.n_exp, l.k_exp, nr, g.route_scale,+ c.ridx + (size_t) r0 * kPicks, c.rw + (size_t) r0 * kPicks, s, l.gate_bias_vl, vl ? &vb : nullptr);+ }+ const int n_vs = L0.n_slots + (int) L0.host_list.size();+ CK(cudaMemcpyAsync(l.vs_dev, l.vs_pin, (size_t) l.n_exp * 4, cudaMemcpyHostToDevice, s)); // (this member's)+ moe_sort(c.ridx, T, l.k_exp, l.vs_dev, n_vs, c.bounds, c.row_tok, c.pos, s);+ CK(cudaMemcpyAsync(c.bounds_pin, c.bounds, (size_t) (n_vs + 1) * 4, cudaMemcpyDeviceToHost, s));+ prof_mark(d, 10);+ // the shared expert into acc while the host waits for the bounds+ if (!gemm(d, c, l.s1, c.x16, D, T, c.sg, FF, 0.0f, err) || !gemm(d, c, l.s3, c.x16, D, T, c.su, FF, 0.0f, err))+ return false;+ swiglu(c.sg, c.su, T * FF, g.lim, c.sg, s);+ x3::rows_f16(c.sg, FF, nullptr, T, FF, c.sh16, s);+ if (!gemm(d, c, l.s2, c.sh16, FF, T, c.acc, D, 0.0f, err)) return false;+ prof_mark(d, 2);+ // the expert stream's copies keep going while the device runs this layer's attention, routing and shared expert:+ // the next items (this layer's likely experts, queued when the layer before routed) go out as they are ready and+ // ring slots free, not only once the bounds are back (Project Maya #360; DS4_FLOW_PUMP_WAIT=0: wait as before)+ static const bool pump_wait = std::getenv("DS4_FLOW_PUMP_WAIT") == nullptr ||+ std::atoi(std::getenv("DS4_FLOW_PUMP_WAIT")) != 0;+ if (c.flow && pump_wait)+ while (cudaStreamQuery(s) == cudaErrorNotReady) {+ if (!flow_pump(c, err)) return false;+ std::this_thread::yield();+ }+ CK(cudaStreamSynchronize(s));+ std::vector<int> b(c.bounds_pin, c.bounds_pin + n_vs + 1);+ if (dump_r != nullptr && mi == 0) {+ const size_t nk = (size_t) T * l.k_exp, np = (size_t) T * kPicks;+ std::vector<int> ri(np), rt(nk);+ std::vector<float> gl((size_t) T * l.n_exp), gb((size_t) l.n_exp);+ CK(cudaMemcpy(ri.data(), c.ridx, np * 4, cudaMemcpyDeviceToHost));+ CK(cudaMemcpy(rt.data(), c.row_tok, nk * 4, cudaMemcpyDeviceToHost));+ CK(cudaMemcpy2D(gl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,+ cudaMemcpyDeviceToHost));+ CK(cudaMemcpy(gb.data(), l.gate_bias, gb.size() * 4, cudaMemcpyDeviceToHost));+ std::lock_guard<std::mutex> lk(dump_mu);+ const int64_t hdr[8] = {l.il, p0, T, l.k_exp, l.n_exp, n_vs, L0.n_slots, kPicks};+ std::fwrite(hdr, 8, 8, dump_r);+ std::fwrite(ri.data(), 4, np, dump_r);+ std::fwrite(L0.vs_pin, 4, (size_t) l.n_exp, dump_r);+ std::fwrite(b.data(), 4, b.size(), dump_r);+ std::fwrite(rt.data(), 4, nk, dump_r);+ std::fwrite(gl.data(), 4, gl.size(), dump_r);+ std::fwrite(gb.data(), 4, gb.size(), dump_r);+ std::fflush(dump_r);+ }+ if (mi == 0 && !l.mtp && (pred_f != nullptr || xin_f != nullptr)) {+ std::lock_guard<std::mutex> lk(dump_mu);+ if (pred_f != nullptr) {+ float* dp = nullptr;+ CK(cudaMalloc(&dp, (size_t) T * 256 * 4));- pred_proj_k<<<T, 256, 0, s>>>(c.x, (int) D, dp);++ strata_ds4_launch(pred_proj_k, T, 256, 0, s, c.x, (int) D, dp);+ std::vector<float> hp((size_t) T * 256), hl((size_t) T * l.n_exp);+ CK(cudaMemcpyAsync(hp.data(), dp, hp.size() * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaMemcpy2DAsync(hl.data(), (size_t) l.n_exp * 4, c.glog, 1024 * 4, (size_t) l.n_exp * 4, (size_t) T,+ cudaMemcpyDeviceToHost, s));+ CK(cudaStreamSynchronize(s));+ CK(cudaFree(dp));+ std::vector<uint16_t> o(hp.size() + hl.size());+ for (size_t i = 0; i < hl.size(); ++i) o[i] = strata::kernels::f16_from_f32(hl[i]);+ for (size_t i = 0; i < hp.size(); ++i) o[hl.size() + i] = strata::kernels::f16_from_f32(hp[i]);+ const uint8_t hd[3] = {(uint8_t) l.il, (uint8_t) (T & 255), (uint8_t) (T >> 8)};+ std::fwrite(hd, 1, 3, pred_f);+ std::fwrite(o.data(), 2, o.size(), pred_f);+ }+ if (xin_f != nullptr && xin_got[(size_t) l.il] < xin_n) { // up to 32 evenly spaced tokens of the chunk+ const int tk = std::min(std::min(T, 32), xin_n - xin_got[(size_t) l.il]);+ const int stride = std::max(1, T / tk);+ std::vector<float> hx((size_t) T * D);+ CK(cudaMemcpyAsync(hx.data(), c.x, hx.size() * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaStreamSynchronize(s));+ const uint8_t hd[3] = {(uint8_t) l.il, (uint8_t) (tk & 255), (uint8_t) (tk >> 8)};+ std::fwrite(hd, 1, 3, xin_f);+ for (int i = 0; i < tk; ++i) std::fwrite(hx.data() + (size_t) std::min(T - 1, i * stride) * D, 4, (size_t) D, xin_f);+ xin_got[(size_t) l.il] += tk;+ }+ }+ if (mi == 0 && trace_f != nullptr && !l.mtp) { // DS4_TRACE: a chunk's picks, u8 (layer | 0x80), u16 T, T x k int16 ids+ std::vector<int> ri((size_t) T * kPicks);+ CK(cudaMemcpy(ri.data(), c.ridx, ri.size() * 4, cudaMemcpyDeviceToHost));+ std::lock_guard<std::mutex> lk(dump_mu);+ const uint8_t hd[3] = {(uint8_t) (l.il | 0x80), (uint8_t) (T & 255), (uint8_t) (T >> 8)};+ std::fwrite(hd, 1, 3, trace_f);+ for (int t = 0; t < T; ++t)+ for (int i = 0; i < l.k_exp; ++i) {+ const int16_t v = (int16_t) ri[(size_t) t * kPicks + i];+ std::fwrite(&v, 2, 1, trace_f);+ }+ }+ if (mi == 0) { // the profile (the CPU lane's experts: by the plan's order)+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;+ for (int e = 0; e < l.n_exp; ++e) {+ const int v = L0.vs_pin[e];+ const int cnt = b[(size_t) v + 1] - b[(size_t) v];+ pr[e] += (uint32_t) cnt;+ if (cnt > 0) { // (the expert cache learns from a chunked prompt as from windows: heat, and the CPU lane's hits)+ l.heat[(size_t) e] += (float) cnt;+ if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);+ }+ }+ st.experts += (int64_t) T * l.k_exp;+ }+ if (c.flow) {+ if (!flow_layer(d, c, L0, mi, wp.k, T, b, err)) return false;+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));+ prof_mark(d, 3);+ return true;+ }+ // the streamed experts as their batches arrive (each batch's slots freed for the copies at once), the VRAM ones+ // in shares between them (the device works while the copies run)+ const int di = l.dev, P = Chunk::kParts;+ int nb = 0;+ for (size_t q = S.kp; q < S.b.size() && S.b[q].il == L0.il && S.b[q].k == wp.k; ++q) ++nb;+ int v_done = 0;+ for (int bi = 0; bi < nb; ++bi) {+ const Batch& bt = S.b[S.kp];+ const int half = (int) (S.kp % P);+ CK(cudaStreamWaitEvent(s, c.filled[half], 0));+ const int v0 = L0.n_slots + bt.j0;+ if (!run_set(d, c, l, c.ring + (size_t) half * (c.ring_n / P) * c.ring_stride, v0, c.ring_stride, c.bounds,+ b.data(), v0, v0 + bt.n, T, err))+ return false;+ CK(cudaEventRecord(c.freed[half], s));+ ++S.kp;+ if (!copy_ahead(d, c, S, di, err)) return false;+ const int v_to = (int) ((int64_t) L0.n_slots * (bi + 1) / nb);+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, v_to, T, err)) return false;+ v_done = v_to;+ }+ if (!run_set(d, c, l, l.vram_base, 0, unit(l), c.bounds, b.data(), v_done, L0.n_slots, T, err)) return false;+ if (nm > 1) allreduce(c.acc, T * D, ab, mi, ++c.arb_seq, d.xcnt, s);+ hc_post_inplace(c.acc, c.h, c.f_post, c.f_comb, D, H, T, s);+ CK(cudaMemcpyAsync(c.pre, c.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));+ prof_mark(d, 3);+ return true;+ }++ // the CPU lane's experts of every layer, in index order, and each expert's virtual slot (before a chunk)+ void plan_streams() {+ for (Layer& l : L) {+ if (l.mtp || l.vs_pin == nullptr) continue;+ l.host_list.clear();+ for (int e = 0; e < l.n_exp; ++e)+ if (l.slot[(size_t) e] < 0) l.host_list.push_back(e);+ for (int m = 0; m < l.nm; ++m) { // (a group's members: each its own copy)+ Layer& x = member(l, m);+ int j = 0;+ for (int e = 0; e < l.n_exp; ++e) x.vs_pin[e] = l.slot[(size_t) e] >= 0 ? l.slot[(size_t) e] : l.n_slots + j++;+ }+ }+ }++ // a chunk's hand-off between groups: h | pre | topk | cand of a chunk of T rows+ Xfer chunk_xfer(uint8_t* host, Chunk& c, int T, bool put) const {+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,+ nc = (size_t) T * cand_stride() * 4;+ void* dv[4] = {c.h, c.pre, c.topk, c.cand};+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};+ const size_t nb[4] = {nh, np, nk, nc};+ Xfer x;+ x.n = 4;+ for (int i = 0; i < 4; ++i) {+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];+ x.dst[i] = put ? (void*) hv[i] : dv[i];+ x.bytes[i] = nb[i];+ }+ return x;+ }++ // a run of chunks (start positions p0[k], sizes T[k], seqs seq[k]) on device di: its layers chunk after chunk, the+ // groups pipelined (a group takes chunk k + 1 while the next takes chunk k: the hand-offs and the sources' rows+ // two deep, each slot reused once its readers acknowledged it), the streamed experts' copies running ahead across+ // layers and chunks+ struct Run {+ std::vector<int64_t> p0;+ std::vector<int> T;+ std::vector<unsigned> seq;+ int64_t snap_pos = -1; // feed_mark: the position whose state before it the layers keep (-1: none)+ // a long prompt's skip (forward_chunks): the cut at row cut_r of chunk cut_k - the chunks before it go through+ // the layers before `yoco`, that layer as layer_chunk's kv_only and none after it; chunk cut_k's rows before+ // cut_r the same, its rows from cut_r on and the chunks after it through every layer (yoco -1: no skip)+ int yoco = -1, cut_k = 0, cut_r = 0;+ bool runs(int k, int il) const { return yoco < 0 || k >= cut_k || il <= yoco; }+ bool kv_only(int k, int il) const { return yoco >= 0 && k < cut_k && il == yoco; }+ int r0(int k, int il) const { return yoco >= 0 && k == cut_k && il >= yoco ? cut_r : 0; } // the first row+ };+ // layer L0 over chunk k per a long prompt's skip (Run): its kv_only, or its rows from Run::r0 on - the chunk's+ // buffers seen from that row (the yoco layer: its kv_only over the rows before it first)+ bool run_layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, Streamer& S, std::string& err,+ const Run& run, int k, int il) {+ if (run.kv_only(k, il)) return layer_chunk(L0, mi, p0, T, wp, S, err, true);+ const int r0 = run.r0(k, il);+ if (r0 <= 0) return layer_chunk(L0, mi, p0, T, wp, S, err);+ if (il == run.yoco) {+ WinPlan w0 = wp;+ w0.snap_t = -1;+ if (!layer_chunk(L0, mi, p0, r0, w0, S, err, true)) return false;+ }+ Chunk& c = *chunks[(size_t) member(L0, mi).dev];+ float* const h0 = c.h;+ float* const pre0 = c.pre;+ c.h += (size_t) r0 * g.hc * g.dim;+ c.pre += (size_t) r0 * 8;+ WinPlan w1 = wp;+ w1.snap_t = wp.snap_t >= r0 ? wp.snap_t - r0 : -1;+ const bool ok = layer_chunk(L0, mi, p0 + r0, T - r0, w1, S, err);+ c.h = h0;+ c.pre = pre0;+ return ok;+ }++ // layer-major (c.lm: a run of several chunks, device di every trunk layer, the flow set up by run_dev_chunks):+ // layer after layer, each through all the chunks in order - a chunk's state to host memory after each of its+ // layers and back before the next - so a layer's experts are read from the files once for the whole run (the+ // layer cache keeps what a chunk read for the chunks after it), where chunk after chunk reads them once a chunk+ bool run_dev_layers(int di, const Run& run, std::string& err) {+ Dev& d = devs[(size_t) di];+ Chunk& c = *chunks[(size_t) di];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();+ Streamer S; // (none: the flow streams)+ const size_t hb = chunk_hand_bytes();+ // the Engram rows: (layer, chunk) after (layer, chunk) in the run's order, each read on a helper thread while+ // the one before runs (slot u % 2, once unit u - 2's copy of it is done)+ std::vector<std::pair<int, int>> egu; // (Engram layer, chunk)+ for (int il : c.fl_layers)+ if (L[(size_t) il].eg >= 0)+ for (int k = 0; k < nk; ++k)+ if (run.runs(k, il)) egu.push_back({L[(size_t) il].eg, k});+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);+ auto read_eg = [this, &c, &run, &egu, eg_per](size_t u) {+ if (u >= 2) cudaEventSynchronize(c.eg_ev[u % 2]);+ const int li = egu[u].first, k = egu[u].second;+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[u % 2] + (size_t) li * c.C * eg_per);+ };+ size_t eg_u = 0;+ std::future<void> eg_next;+ if (!egu.empty()) eg_next = std::async(std::launch::async, read_eg, (size_t) 0);+ for (size_t li = 0; li < c.fl_layers.size(); ++li) {+ const int il = c.fl_layers[li];+ Layer& L0 = L[(size_t) il];+ const bool last_l = li + 1 == c.fl_layers.size();+ for (int k = 0; k < nk; ++k) {+ if (!run.runs(k, il)) continue; // (a skipped early chunk: nothing after the yoco layer)+ if (halt_now()) return halt_end(di, err); // (a Stop: one GPU, this one)+ const int64_t p0 = run.p0[(size_t) k];+ const int T = run.T[(size_t) k], slot = k % 2;+ const bool last_k = k + 1 == nk;+ WinPlan wp;+ wp.par = true;+ wp.async = true;+ wp.win = run.seq[(size_t) k];+ wp.k = k;+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;+ if (L0.eg >= 0) { // this unit's rows ready, the next one's on their way+ const double te = now_ms();+ eg_next.get();+ st.engram_ms += now_ms() - te;+ wp.eg_slot = (int) (eg_u % 2);+ if (++eg_u < egu.size()) eg_next = std::async(std::launch::async, read_eg, eg_u);+ }+ if (li == 0) { // the chunk's input: its embedding rows+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));+ for (int t = 0; t < T; ++t)+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),+ (size_t) D * 2);+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));+ CK(cudaEventRecord(c.emb_ev[slot], s));+ embed(c.emb, T, c.h, D, H, s);++ if (std::getenv("DS4_DBG_EMB")) {++ ds4_metal_flush();++ float *hp = c.h;++ float hh[4] = {0,0,0,0};++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);++ }+ init_pre(c.pre, T, s);+ } else { // or its state after the layer before+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, false);+ for (int i = 0; i < x.n; ++i)+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyHostToDevice, s));+ }+ if (last_k && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);+ if (it != g.d_targets.end()) {+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();+ const int nr = std::min(T, tap_cap);+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);+ CK(cudaStreamWaitEvent(s, ev_seed, 0));+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,+ (size_t) nr, cudaMemcpyDeviceToHost, s));+ }+ }+ const bool kvo = run.kv_only(k, il);+ if (!run_layer(L0, 0, p0, T, wp, S, err, run, k, il)) return false;+ if (dump_l != nullptr) dump_layer(d, c.h, il, p0, T);+ if (kvo) continue; // (no layer after it reads this chunk's state)+ if (!last_l) { // the chunk's state kept for the next layer+ const Xfer x = chunk_xfer(c.lm_state + (size_t) k * hb, c, T, true);+ for (int i = 0; i < x.n; ++i)+ CK(cudaMemcpyAsync(x.dst[i], x.src[i], x.bytes[i], cudaMemcpyDeviceToHost, s));+ continue;+ }+ // through every layer: the chunk's end, and the logits+ if ((size_t) k < chunk_done.size()) {+ CK(cudaEventRecord(chunk_done[(size_t) k], s));+ chunk_rec[(size_t) k].store(1, std::memory_order_release);+ }+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;+ for (int r0 = 0; r0 < T; r0 += kWin) {+ const int nr = std::min(kWin, T - r0);+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);+ if (!head_share(d, d.mi, d.x, nr, err)) return false;+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaStreamSynchronize(s));+ }+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)+ const int t0 = T - n_logits;+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;+ }+ }+ lm_layers.store((int) li + 1, std::memory_order_release);+ }+ return true;+ }+ bool run_dev_chunks(int di, const Run& run, std::string& err) {+ Dev& d = devs[(size_t) di];+ Chunk& c = *chunks[(size_t) di];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ const int D = g.dim, H = g.hc, nk = (int) run.T.size();+ Streamer S;+ const int B = std::max(1, c.ring_n / Chunk::kParts);+ if (c.flow) { // (the flow decides its streams as the layers route; nothing planned here)+ CK(cudaStreamSynchronize(s));+ CK(cudaStreamSynchronize(c.cs));+ c.items.clear();+ c.base = c.n_read = c.n_dma = c.n_comp = 0;+ c.n_file = 0;+ std::fill(c.stage_item.begin(), c.stage_item.end(), -1);+ c.fl_layers.clear();+ for (int il = 0; il < g.nl; ++il) // (a group's layers: on each member)+ for (int m = 0; m < L[(size_t) il].nm; ++m)+ if (member(L[(size_t) il], m).dev == di) c.fl_layers.push_back(il);+ c.fl_T = run.T;+ c.fl_yoco = run.yoco;+ c.fl_cut_k = run.cut_k;+ c.fl_cut_r = run.cut_r;+ c.spec_k = c.spec_il = -1;+ c.wait_ms = 0;+ c.it_file = c.it_ram = c.it_spec = c.it_spec_unused = c.it_lc = 0;+ c.lc_il = -1;+ c.lc_used = 0;+ c.lc_first = c.lc_chk = 0;+ c.io_d0 = c.io->direct_bytes();+ c.io_b0 = c.io->buffered_bytes();+ c.io_busy0 = c.io->busy_ms();+ if (!c.fl_layers.empty()) {+ flow_spec(c, 0, c.fl_layers[0], run.T[0]);+ if (!flow_pump(c, err)) return false;+ }+ if (c.lm) return run_dev_layers(di, run, err);+ }+ for (int k = 0; k < nk && !c.flow; ++k)+ for (int il = 0; il < g.nl; ++il) {+ if (!run.runs(k, il) || run.kv_only(k, il)) continue; // (a long prompt's skip: no experts there)+ const Layer& L0 = L[(size_t) il];+ bool here = false;+ for (int m = 0; m < L0.nm; ++m) here = here || member(L0, m).dev == di;+ if (!here) continue;+ for (int j0 = 0; j0 < (int) L0.host_list.size(); j0 += B)+ S.b.push_back({k, il, j0, std::min(B, (int) L0.host_list.size() - j0)});+ }+ if (!S.b.empty() && c.ring == nullptr) {+ err = "no slots to stream experts into on CUDA" + std::to_string(d.id);+ return false;+ }+ std::atomic<bool> ra_stop{false};+ std::thread ra;+ if (file_experts && !S.b.empty()) ra = std::thread([this, &S, di, &ra_stop] { read_ahead(S, di, ra_stop); });+ struct Join { // (every way out of the run: the read-ahead ends before S does)+ std::atomic<bool>& stop;+ std::thread& th;+ ~Join() {+ stop.store(true);+ if (th.joinable()) th.join();+ }+ } ra_join{ra_stop, ra};+ if (!copy_ahead(d, c, S, di, err)) return false;+ // this device's Engram layers: chunk k's rows read on a helper thread while chunk k - 1 runs (slot k % 2, once+ // chunk k - 2's copy of it is done)+ std::vector<int> egs;+ for (const Layer& l0 : L)+ for (int m = 0; m < l0.nm; ++m)+ if (member(l0, m).dev == di && !l0.mtp && l0.eg >= 0) egs.push_back(l0.eg);+ const size_t eg_per = (size_t) (pk.eg_ngram - 1) * pk.eg_heads * (pk.eg_hd + pk.eg_hd / 32);+ auto read_eg = [this, &c, &run, &egs, eg_per](int k) {+ if (k >= 2) cudaEventSynchronize(c.eg_ev[k % 2]);+ for (int li : egs)+ engram_rows(run.p0[(size_t) k], run.T[(size_t) k], li, c.eg_host[k % 2] + (size_t) li * c.C * eg_per);+ };+ std::future<void> eg_next;+ if (!egs.empty()) eg_next = std::async(std::launch::async, read_eg, 0);+ for (int k = 0; k < nk; ++k) {+ if (!chunk_go(k)) return halt_end(di, err); // (a Stop: the chunks every device reaches are done)+ const int64_t p0 = run.p0[(size_t) k];+ const int T = run.T[(size_t) k], slot = k % 2;+ const bool last_k = k + 1 == nk;+ if (!egs.empty()) { // chunk k's rows ready, chunk k + 1's on their way+ const double te = now_ms();+ eg_next.get();+ if (d.mi == 0) st.engram_ms += now_ms() - te;+ if (k + 1 < nk) eg_next = std::async(std::launch::async, read_eg, k + 1);+ }+ WinPlan wp;+ wp.par = true;+ wp.async = true;+ wp.win = run.seq[(size_t) k];+ wp.k = k;+ wp.snap_t = run.snap_pos >= p0 && run.snap_pos < p0 + T ? (int) (run.snap_pos - p0) : -1;+ bool started = false;+ std::vector<char> got(srcs.size(), 0);+ for (int il = 0; il < g.nl && run.runs(k, il); ++il) {+ if (halt_now()) return halt_end(di, err); // (one GPU: after the layer it is on)+ Layer& L0 = L[(size_t) il];+ int mi = -1;+ for (int m = 0; m < L0.nm; ++m)+ if (member(L0, m).dev == di) mi = m;+ if (mi < 0) continue;+ if (!started) { // the chunk's input: its embedding rows, or the previous group's hand-off+ started = true;+ if (il == 0) {+ if (k >= 2) CK(cudaEventSynchronize(c.emb_ev[slot]));+ for (int t = 0; t < T; ++t)+ std::memcpy(c.emb_host[slot] + (size_t) t * D, emb_row(p0 + t, toks[(size_t) (p0 + t)]),+ (size_t) D * 2);+ CK(cudaMemcpyAsync(c.emb, c.emb_host[slot], (size_t) T * D * 2, cudaMemcpyHostToDevice, s));+ CK(cudaEventRecord(c.emb_ev[slot], s));+ embed(c.emb, T, c.h, D, H, s);++ if (std::getenv("DS4_DBG_EMB")) {++ ds4_metal_flush();++ float *hp = c.h;++ float hh[4] = {0,0,0,0};++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);++ }+ init_pre(c.pre, T, s);+ } else {+ Chunk& a = *chunks[(size_t) L[(size_t) il - 1].dev];+ xfer_get(chunk_xfer(a.hand[slot], c, T, false), a.hand_flag[slot], wp.win, s);+ set_flag(a.hand_ack + ((size_t) slot * 8 + mi) * 32, wp.win, s);+ }+ }+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {+ if (sidx < 0 || got[(size_t) sidx]) continue;+ got[(size_t) sidx] = 1;+ Src& sr = srcs[(size_t) sidx];+ if (L[(size_t) sr.layer].grp == L0.grp) continue;+ const int64_t g_first = p0 / sr.ratio;+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);+ Xfer x;+ if (ng > 0 && sr.ckv[(size_t) di] != nullptr) {+ x.src[x.n] = sr.stage(slot);+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();+ x.bytes[x.n++] = (size_t) ng * ckv_rb();+ }+ if (ng > 0 && sr.ik[(size_t) di] != nullptr) {+ x.src[x.n] = sr.stage(slot) + x_cap() * ckv_rb();+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();+ x.bytes[x.n++] = (size_t) ng * ik_rb();+ }+ if (x.n > 0) xfer_get(x, sr.flag(slot), wp.win, s);+ set_flag(sr.x_ack + ((size_t) slot * devs.size() + di) * 32, wp.win, s); // (taken, or none)+ }+ if (last_k && mi == 0 && g.n_mtp > 0) { // the drafter's taps: the last chunk's last rows+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);+ if (it != g.d_targets.end()) {+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();+ const int nr = std::min(T, tap_cap);+ hc_mean(c.h + (size_t) (T - nr) * H * D, nr, H, D, d.tap, D, s);+ CK(cudaStreamWaitEvent(s, ev_seed, 0));+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4,+ (size_t) nr, cudaMemcpyDeviceToHost, s));+ }+ }+ if (!run_layer(L0, mi, p0, T, wp, S, err, run, k, il)) return false;+ if (dump_l != nullptr && mi == 0) dump_layer(d, c.h, il, p0, T);+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;+ if (last && mi == 0 && il + 1 < g.nl) { // the hand-off, once the next group took the chunk before last+ const int nnext = L[(size_t) il + 1].nm;+ if (k >= 2) wait_flags(c.hand_ack + (size_t) slot * 8 * 32, nnext, 32, wp.win - 2, s);+ xfer_put(chunk_xfer(c.hand[slot], c, T, true), c.hand_flag[slot], wp.win, d.xcnt, s);+ }+ }+ if (di == head_dev && (size_t) k < chunk_done.size()) {+ CK(cudaEventRecord(chunk_done[(size_t) k], s));+ chunk_rec[(size_t) k].store(1, std::memory_order_release);+ }+ if (d.grp == devs[(size_t) head_dev].grp) { // the head group: each member its share of the logits+ if (logits_all) { // every row's logits (tests): the head over the chunk's rows, 16 at a time+ const size_t base = (size_t) (p0 - run.p0[0]) * g.vocab;+ for (int r0 = 0; r0 < T; r0 += kWin) {+ const int nr = std::min(kWin, T - r0);+ hc_pre_norm(c.h + (size_t) r0 * H * D, c.pre + (size_t) r0 * 8, d.norm_w, D, H, nr, g.eps, d.x, s);+ if (!head_share(d, d.mi, d.x, nr, err)) return false;+ if (di != head_dev) continue;+ CK(cudaMemcpyAsync(chunk_logits.data() + base + (size_t) r0 * g.vocab, d.logits,+ (size_t) nr * g.vocab * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaStreamSynchronize(s));+ }+ } else if (last_k && n_logits > 0) { // (the last rows' logits, into the window path's buffers)+ const int t0 = T - n_logits;+ hc_pre_norm(c.h + (size_t) t0 * H * D, c.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, s);+ if (!head_share(d, d.mi, d.x, n_logits, err)) return false;+ }+ }+ }+ return true;+ }++ // ---- a prompt's path: the flow or windows of 16+ // this PC's speeds for pick_path (every device on the flow, the experts in place): the DMA from pinned staging to a+ // GPU; the readers copying RAM into staging; the disk read by them (experts not in RAM); the CPU lane on experts in+ // RAM and on ones it reads from the disk first - each timed once, a few experts' worth (DS4_PATH_PICK=0: always the+ // flow, as before)+ void calibrate_paths() {+ if (chunk <= 0 || chunk_min != kFlowChunkMin || std::getenv("DS4_CHUNK_MIN") != nullptr) return;+ if (const char* v = std::getenv("DS4_PATH_PICK"))+ if (std::atoi(v) == 0) return;+ int di = -1;+ for (size_t i = 0; i < chunks.size() && di < 0; ++i)+ if (chunks[i] != nullptr && chunks[i]->flow && chunks[i]->S >= 2) di = (int) i;+ if (di < 0) return;+ Chunk& c = *chunks[(size_t) di];+ Dev& d = devs[(size_t) di];+ cudaSetDevice(d.id);+ const double t0 = now_ms();+ // the DMA: staging -> the ring (once to warm up, once timed)+ const size_t nb = std::min({(size_t) 256 << 20, (size_t) c.S * c.stage_stride, (size_t) c.ring_n * c.ring_stride});+ double ms = 0;+ for (int rep = 0; rep < 2; ++rep) {+ const double a = now_ms();+ if (cudaMemcpyAsync(c.ring, c.stage, nb, cudaMemcpyHostToDevice, c.cs) != cudaSuccess ||+ cudaStreamSynchronize(c.cs) != cudaSuccess) {+ cudaGetLastError();+ return;+ }+ ms = now_ms() - a;+ }+ pm.dma = (double) nb / std::max(1e-3, ms / 1000.0);+ // the readers: RAM -> staging (a buffer of its own, touched first), in as many jobs as they are+ auto run_jobs = [&](std::vector<IoQueue::Job>& jobs, std::atomic<int>& left) {+ const double a = now_ms();+ left.store((int) jobs.size());+ c.io->push(jobs.data(), (int) jobs.size());+ while (left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);+ return now_ms() - a;+ };+ {+ const size_t part = std::min(nb, (size_t) 64 << 20) / 8 / 4096 * 4096;+ std::vector<uint8_t> src(part * 8, 1);+ std::vector<IoQueue::Job> jobs(8);+ std::atomic<int> left{0};+ std::atomic<bool> bad{false};+ for (int i = 0; i < 8; ++i) {+ jobs[(size_t) i].mem = src.data() + (size_t) i * part;+ jobs[(size_t) i].dst = c.stage + (size_t) i * part;+ jobs[(size_t) i].len = part;+ jobs[(size_t) i].left = &left;+ jobs[(size_t) i].bad = &bad;+ }+ run_jobs(jobs, left);+ const double t1 = run_jobs(jobs, left), t2 = run_jobs(jobs, left);+ pm.copy = (double) (part * 8) / std::max(1e-3, std::min(t1, t2) / 1000.0);+ }+ // experts not in RAM (none of their sampled pages resident), of the trunk layer with the most CPU-lane ones:+ // up to min(8, staging slots) read by the readers as the flow reads them, then 8 others through the CPU lane+ // three times (from the disk, then in RAM: the faster of two)+ int best = -1, most = 0;+ for (int il = 0; il < g.nl; ++il) {+ int n = 0;+ for (int e = 0; e < L[(size_t) il].n_exp; ++e) n += L[(size_t) il].slot[(size_t) e] < 0;+ if (n > most) most = n, best = il;+ }+ if (best < 0) return;+ const Layer& l = L[(size_t) best];+ std::vector<int> cold;+ const int n_read = std::min(8, c.S), n_cpu = 8;+ for (int e = 0; e < l.n_exp && (int) cold.size() < n_read + n_cpu; ++e) {+ if (l.slot[(size_t) e] >= 0 || (!l.ram.empty() && l.ram[(size_t) e] >= 0)) continue;+ bool res = false;+ for (int r = 0; r < 3; ++r) res = res || os::resident_sampled(mx_mat(l, e, r), mx_part_bytes(l, r), 4);+ if (!res) cold.push_back(e);+ }+ if ((int) cold.size() >= n_read + n_cpu) {+ std::vector<IoQueue::Job> jobs;+ std::atomic<int> left{0};+ std::atomic<bool> bad{false};+ for (int i = 0; i < n_read; ++i) {+ uint8_t* sb = c.stage + (size_t) i * c.stage_stride;+ for (int r = 0; r < 3; ++r) {+ IoQueue::Job j;+ const os::fd_t pf = pk.pfds.empty() ? os::kBadFd : pk.pfds[(size_t) l.mx_file[r]];+ j.fd = pf != os::kBadFd ? pf : pk.fds[(size_t) l.mx_file[r]];+ j.dfd = pk.dfds.empty() ? os::kBadFd : pk.dfds[(size_t) l.mx_file[r]];+ j.off = l.mx_off[r] + (uint64_t) cold[(size_t) i] * l.mx_stride[r];+ j.len = mx_part_bytes(l, r);+ j.dst = sb;+ j.left = &left;+ j.bad = &bad;+ j.map = pk.at(l.mx_file[r], j.off);+ jobs.push_back(j);+ sb += flow_region(r);+ }+ }+ const double rd = run_jobs(jobs, left);+ if (!bad.load()) pm.disk = (double) n_read * (double) l.blob / std::max(1e-3, rd / 1000.0);+ }+ { // the CPU lane: entries of one token (as most of a window's are), from the disk (if any were cold), then RAM+ const size_t first = (int) cold.size() >= n_read + n_cpu ? (size_t) n_read : 0;+ std::vector<Ent> ents;+ for (int i = 0; i < n_cpu; ++i) {+ Ent en;+ en.e = first + (size_t) i < cold.size() ? cold[first + (size_t) i] : -1;+ if (en.e < 0) // (no cold ones: the first CPU-lane experts, wherever they are)+ for (int e = 0, k = 0; e < l.n_exp && en.e < 0; ++e)+ if (l.slot[(size_t) e] < 0 && k++ == i) en.e = e;+ if (en.e < 0) break;+ en.n = 1;+ en.slot0 = i;+ en.tok[0] = 0;+ ents.push_back(en);+ }+ if (ents.empty()) return;+ std::vector<float> x((size_t) g.dim);+ for (size_t i = 0; i < x.size(); ++i) x[i] = 0.02f * std::sin(0.37f * (float) i);+ const double bytes = (double) ents.size() * (double) l.blob;+ double a = now_ms();+ cpu_experts(l, x.data(), ents, (size_t) g.dim);+ const double t_first = now_ms() - a;+ double t_warm = 1e30;+ for (int rep = 0; rep < 2; ++rep) {+ a = now_ms();+ cpu_experts(l, x.data(), ents, (size_t) g.dim);+ t_warm = std::min(t_warm, now_ms() - a);+ }+ pm.cpu = bytes / std::max(1e-3, t_warm / 1000.0);+ if (first > 0 || (int) cold.size() >= n_cpu) pm.cpu_cold = bytes / std::max(1e-3, t_first / 1000.0);+ }+ pm.on = pm.dma > 0 && pm.copy > 0 && pm.cpu > 0;+ if (o.verbose) {+ char dk[32] = "not measured", cc[32] = "not measured";+ if (pm.disk > 0) std::snprintf(dk, sizeof dk, "%.2f GB/s", pm.disk / 1e9);+ if (pm.cpu_cold > 0) std::snprintf(cc, sizeof cc, "%.2f GB/s", pm.cpu_cold / 1e9);+ std::fprintf(stderr,+ "ds4: prompts through the flow or windows by this PC's speeds: DMA %.1f GB/s, RAM to staging "+ "%.1f GB/s, the disk %s; the CPU lane %.1f GB/s in RAM, %s from the disk (%.0f ms)\n",+ pm.dma / 1e9, pm.copy / 1e9, dk, pm.cpu / 1e9, cc, now_ms() - t0);+ }+ }+ // layer l's CPU-lane experts (VRAM holds the rest) and the chance a token picks a given one of them: the profile's+ // share of the layer's picks they took, spread evenly over them, times pm.m (what the flow's runs found)+ double pm_rate(const Layer& l, int& n_cold) const {+ const uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;+ double sum = 0, cold = 0;+ n_cold = 0;+ for (int e = 0; e < l.n_exp; ++e) {+ sum += pr[e];+ if (l.slot[(size_t) e] < 0) cold += pr[e], ++n_cold;+ }+ if (n_cold == 0) return 0.0;+ const double share = sum > 0 && cold > 0 ? cold / sum : (double) n_cold / l.n_exp;+ return pm.m * (double) l.k_exp * share / n_cold;+ }+ // the CPU-lane experts n tokens pick in the trunk's layers, expected (each layer's n_cold (1 - e^(-rate n)))+ double pm_picks(int n) const {+ double d = 0;+ for (int il = 0; il < g.nl; ++il) {+ int nc = 0;+ const double a = pm_rate(L[(size_t) il], nc);+ d += nc * (1.0 - std::exp(-a * n));+ }+ return d;+ }+ // a run of one chunk of n tokens on the flow picked `picked` CPU-lane experts: pm.m to what makes the expectation+ // that (bisection over its log; a running mean of the log)+ void pm_learn(int n, double picked) {+ if (!pm.on || n < 2 || picked < 1) return;+ const double m0 = pm.m;+ double lo = std::log(1e-3), hi = std::log(1e3);+ for (int it = 0; it < 40; ++it) {+ pm.m = std::exp(0.5 * (lo + hi));+ (pm_picks(n) < picked ? lo : hi) = 0.5 * (lo + hi);+ }+ const double obs = 0.5 * (lo + hi);+ pm.m = std::exp(pm.n_m == 0 ? obs : 0.7 * std::log(m0) + 0.3 * obs);+ ++pm.n_m;+ }+ // a prompt of n tokens that the flow could take (n >= chunk_min): true the flow, false windows of 16 - each path's+ // time predicted from the bytes of CPU-lane experts it would move (the flow: each picked one once, n_cold (1 -+ // e^(-rate n)) a layer, pm_rate) or compute (windows: each window's own picks), those in RAM (each layer's share:+ // sampled pages of up to 32 of its CPU-lane experts, mincore / QueryWorkingSetEx; the RAM tier's) and those not at+ // this PC's speeds - the flow's times what it really took against its predictions so far for prompts of that size+ // (pm_k), windows' once some ran what a window took times the windows; the flow unless windows are predicted+ // faster (by half, with less than kPickRes in RAM); said with --verbose+ // The choice is made from the prompt's length alone (pick_flow_min): the same prompt takes the same path - and so+ // gives the same bits, the two paths round differently - in every session and on every restart. The timing prediction below+ // (DS4_PICK=adaptive) used measured speeds and sampled page residency, so it flipped between runs: r9700, one 16-token feed+ // through the flow in one run, another feed in the next, three different logits checksums in six runs. Measured on 4 and 1 R9700+ // (first prompt of a run, windows against the flow, ms): 28 tokens 594 / 4966 (4 cards), 1094 / 5588 (1 card); 100: 1164 / 5375,+ // 3279 / 7299; 300: 2831 / 5874, 9018 / 9803; 600: 5304 / 6808, 16969 / 11327; 1500: 13325 / 8012, 41547 / 18187 - the flow pays a+ // fixed ~5 s the first time (its buffers and GEMM libraries), then wins from about 400 tokens up.+ // The length from which the flow is taken: one number for every box (ds4_pick_rule.hpp says why 96), DS4_PICK_MIN=N overrides.+ int pick_flow_min() {+ if (pick_min_ > 0) return pick_min_;+ pick_min_ = prompt_flow_min(std::getenv("DS4_PICK_MIN"));+ std::fprintf(stderr, "ds4: prompt path: the chunked pass from %d tokens, windows of 16 below\n", pick_min_);+ return pick_min_;+ }+ int pick_min_ = 0;+ bool pick_path(int n) {+ static const bool adaptive = [] {+ const char* e = std::getenv("DS4_PICK");+ return e != nullptr && std::strcmp(e, "adaptive") == 0;+ }();+ pm.last = -1;+ if (!adaptive) return n >= pick_flow_min();+ if (!pm.on || n >= kPickMax) return true;+ double f_res = 0, f_cold = 0, w = 0;+ const int nw = n / kWin, rem = n % kWin;+ for (int il = 0; il < g.nl; ++il) {+ const Layer& l = L[(size_t) il];+ int nc = 0;+ const double a = pm_rate(l, nc);+ if (nc == 0) continue;+ int sampled = 0, res = 0, seen = 0;+ const int step = std::max(1, nc / 32);+ for (int e = 0; e < l.n_exp; ++e) {+ if (l.slot[(size_t) e] >= 0 || seen++ % step != 0) continue;+ bool r = !l.ram.empty() && l.ram[(size_t) e] >= 0;+ if (!r) {+ r = true;+ for (int p = 0; p < 3 && r; ++p) r = os::resident_sampled(mx_mat(l, e, p), mx_part_bytes(l, p), 2);+ }+ ++sampled;+ res += r;+ }+ const double share = sampled > 0 ? (double) res / sampled : 0.0, b = (double) l.blob;+ auto u = [&](int t) { return nc * (1.0 - std::exp(-a * t)); };+ f_res += share * u(n) * b;+ f_cold += (1.0 - share) * u(n) * b;+ w += ((double) nw * u(kWin) + (rem > 0 ? u(rem) : 0.0)) * b;+ }+ const double r_fr = std::min(pm.copy, pm.dma), r_fc = pm.disk > 0 ? std::min(pm.disk, pm.dma) : r_fr;+ const double r_wc = pm.cpu_cold > 0 ? pm.cpu_cold : 1.0 / (1.0 / r_fc + 1.0 / pm.cpu);+ const double t_flow = f_res / r_fr + f_cold / r_fc;+ const double t_win = std::max(0.0, w - f_cold) / pm.cpu + f_cold / r_wc;+ const int cls = pm_cls(n), n_win = (n + kWin - 1) / kWin;+ // (windows: once some ran, what a window took on average times the windows the prompt needs - each pays the+ // fixed costs of every layer on every GPU again, which a pass of the flow pays once)+ const double pf = pm_k(cls) * t_flow, pw = pm.win_n > 0 ? pm.win_s * n_win : t_win;+ // (the flow unless windows are predicted faster - in half its time at least, unless kPickRes of the CPU-lane+ // experts' bytes are in RAM: the flow wins on every PC whose experts come from an SSD, where a wrong pick would+ // cost the most; windows win where they are all in RAM)+ const double res_share = f_res / std::max(1.0, f_res + f_cold);+ const bool flow = !(pw < pf && (res_share >= kPickRes || 2.0 * pw <= pf));+ pm.last = flow ? 1 : 0;+ pm.last_cls = cls;+ pm.last_nw = n_win;+ pm.last_pred = flow ? t_flow : t_win;+ if (o.verbose)+ std::fprintf(stderr,+ "ds4: a prompt of %d tokens through %s: predicted %.2f s, %s %.2f s (%.0f%% of its CPU-lane "+ "experts' %.1f GB in RAM)\n",+ n, flow ? "the flow" : "windows", flow ? pf : pw, flow ? "windows" : "the flow", flow ? pw : pf,+ 100.0 * res_share, (f_res + f_cold) / 1e9);+ return flow;+ }+ // a prompt's size class for pm.k: 2-31, 32-127, 128-511, 512 tokens and more+ static int pm_cls(int n) { return n < 32 ? 0 : n < 128 ? 1 : n < 512 ? 2 : 3; }+ // the flow's time factor for size class c: its own, else its nearest class's, else 1 (never the windows': theirs+ // is mostly the fixed costs a window pays again and again)+ double pm_k(int c) const {+ for (int dc = 0; dc < 4; ++dc)+ for (int cc : {c - dc, c + dc})+ if (cc >= 0 && cc < 4 && pm.kn[0][cc] > 0) return pm.k[0][cc];+ return 1.0;+ }+ // n tokens through windows of 16, the last one's logits_rows rows of logits+ bool feed_windows(const int* tokens, int n, int logits_rows, std::string& err) {+ logits_all = false;+ for (int a = 0; a < n; a += kWin) {+ const int T = std::min(kWin, n - a);+ if (!forward(tokens + a, T, a + T == n ? logits_rows : 0, err)) return false;+ }+ return true;+ }+ // the picked path took ms: the flow's predictions for the prompt's size class scaled by what it really takes (taken+ // / predicted), windows' time a window - running means+ void path_done(double ms) {+ if (pm.last < 0 || pm.last_pred <= 0) return;+ if (pm.last == 0) {+ const double w = ms / 1000.0 / std::max(1, pm.last_nw);+ pm.win_s = pm.win_n == 0 ? w : 0.7 * pm.win_s + 0.3 * w;+ ++pm.win_n;+ } else {+ const double ratio = std::min(8.0, std::max(0.25, ms / 1000.0 / pm.last_pred));+ double& k = pm.k[0][pm.last_cls];+ int64_t& n = pm.kn[0][pm.last_cls];+ k = n == 0 ? ratio : 0.7 * k + 0.3 * ratio;+ ++n;+ }+ pm.last = -1;+ }+ // a run layer-major? - one device holding every layer on the flow, several chunks, and what the chunks after the+ // first would read from the files again more than the chunks' states moved to host memory and back at every layer+ // (DS4_LAYER_MAJOR=0: never); then its host memory: the chunks' states (pinned, else pageable) and the layer cache,+ // the largest layer's experts read from the files if RAM allows (ram_avail() less kRamMargin; DS4_LAYER_CACHE_MB+ // sets it) - at least one expert, or not layer-major+ bool lm_begin(const Run& run) {+ static const char* env = std::getenv("DS4_LAYER_MAJOR");+ if (devs.size() != 1 || run.T.size() < 2 || (env != nullptr && std::atoi(env) == 0)) return false;+ Chunk& c = *chunks[0];+ if (!c.flow) return false;+ size_t need = 0, n_exp = 0, blob_max = 0;+ double file_b = 0;+ for (int il = 0; il < g.nl; ++il) {+ const Layer& l = L[(size_t) il];+ size_t ne = 0;+ for (int e : l.host_list)+ if (l.ram.empty() || l.ram[(size_t) e] < 0) ++ne;+ const size_t stride = (l.blob + 63) & ~(size_t) 63;+ need = std::max(need, ne * stride);+ n_exp = std::max(n_exp, (size_t) l.n_exp);+ blob_max = std::max(blob_max, stride);+ file_b += (double) ne * (double) l.blob;+ }+ int64_t rows = 0;+ for (int T : run.T) rows += T;+ const size_t hb = chunk_hand_bytes(), state_b = run.T.size() * hb;+ const double moved = 2.0 * (double) rows * ((double) hb / (double) chunk) * (double) g.nl;+ if (need == 0 || (double) (run.T.size() - 1) * file_b <= moved) return false;+ size_t lc_bytes = need;+ if (const char* mb = std::getenv("DS4_LAYER_CACHE_MB")) lc_bytes = std::min(need, (size_t) std::atoll(mb) << 20);+ else {+ const uint64_t avail = ram_avail();+ if (avail != ~uint64_t{0})+ lc_bytes = std::min<uint64_t>(need, avail > kRamMargin + state_b ? avail - kRamMargin - state_b : 0);+ }+ if (lc_bytes < blob_max) return false;+ void* st_p = nullptr;+ c.lm_pinned = cudaHostAlloc(&st_p, state_b, cudaHostAllocPortable) == cudaSuccess;+ if (!c.lm_pinned) {+ cudaGetLastError();+ st_p = os::anon_alloc(state_b, -1, 1);+ if (st_p == nullptr) return false;+ }+ c.lc = (uint8_t*) os::anon_alloc(lc_bytes, -1, 1);+ if (c.lc == nullptr) {+ if (c.lm_pinned) cudaFreeHost(st_p);+ else os::anon_free(st_p, state_b);+ return false;+ }+ c.lm_state = (uint8_t*) st_p;+ c.lm_state_bytes = state_b;+ c.lc_bytes = lc_bytes;+ c.lc_of.assign(n_exp, -1);+ c.lc_writer.assign(n_exp, -1);+ c.lm = true;+ lm_layers.store(0);+ return true;+ }+ void lm_end() {+ for (size_t di = 0; di < chunks.size(); ++di) {+ Chunk& c = *chunks[di];+ if (!c.lm) continue;+ cudaSetDevice(devs[di].id);+ cudaStreamSynchronize(devs[di].st); // (nothing of the run's still reads or writes them: the device's+ cudaStreamSynchronize(c.cs); // copies done, every read the run issued done - a failed run's too)+ for (size_t i = c.base; i < c.n_read; ++i)+ while (fit(c, i).left.load(std::memory_order_acquire) != 0) c.io->wait_any(500);+ if (c.lm_pinned) cudaFreeHost(c.lm_state);+ else os::anon_free(c.lm_state, c.lm_state_bytes);+ os::anon_free(c.lc, c.lc_bytes);+ c.lm_state = c.lc = nullptr;+ c.lm_state_bytes = c.lc_bytes = 0;+ c.lm = c.lm_pinned = false;+ }+ }++ // the first layer of a long prompt's second half (forward_chunks' two passes), or -1: the first with compress ratio 1+ // (V4.1: 20), every one after it ratio 1 too and owning no compressed KV or index keys of its own (they read its),+ // every trunk layer on one GPU. The settings' MAYA_DS4_PROMPT_SKIP (STRATA_DS4_PROMPT_SKIP; DS4_YOCO for tests):+ // 1 (the default) the skip, 2 DeepSeek's own serving's (the last 128 rows only: faster, the answers shift a+ // little), 0 off+ static int yoco_mode() {+ static const int m = [] {+ const char* v = std::getenv("DS4_YOCO");+ if (v == nullptr) v = std::getenv("STRATA_DS4_PROMPT_SKIP");+ return v != nullptr ? std::atoi(v) : 1;+ }();+ return m;+ }+ int yoco_layer() const {+ if (yoco_mode() == 0 || chunk <= 0 || g.nl < 2 || (int) g.ratios.size() < g.nl) return -1;+ int yl = -1;+ for (int il = 0; il < g.nl && yl < 0; ++il)+ if (g.ratios[(size_t) il] == 1) yl = il;+ if (yl <= 0 || !L[(size_t) yl].kv_src) return -1;+ for (int il = 0; il < g.nl; ++il) {+ const Layer& l = L[(size_t) il];+ if (l.nm != 1 || l.dev != L[0].dev) return -1;+ if (il > yl && (g.ratios[(size_t) il] != 1 || l.kv_src)) return -1;+ }+ return yl;+ }++ // n tokens at positions pos .. in chunks of equal size (at most `chunk`), pipelined over the groups; the logits of+ // the last min(logits_rows, 16) rows (logits_rows > 16: a run of one chunk, every row's into chunk_logits)+ bool forward_chunks(const int* tk, int n, int logits_rows, std::string& err, int snap_slot = -1) {+ if (n < 1) return true;+ flow_used.store(true);+ std::lock_guard<std::mutex> warm_lk(warm_mu); // (the warm-up, if it is running, finishes first)+ if (pos + n > o.max_context) {+ err = "the context is full";+ return false;+ }+ for (int t = 0; t < n; ++t)+ if (tk[t] < 0 || tk[t] >= g.vocab) {+ err = "token " + std::to_string(tk[t]) + " out of range";+ return false;+ }+ // a long prompt's skip: the layers from yoco_layer() on (V4.1: 20-39) read the earlier positions only through+ // that layer's compressed KV and index keys, and each looks back one window (win - 1 positions) more than the+ // layer before - so they need their own work only on the last keep = (nl - yoco) (win - 1) + win rows for the+ // run's last rows and the rings the decode reads to be what the full work gives (up to the experts' summing+ // order). The chunks stay as they are: the ones before the cut go through the layers before yoco and its+ // kv_only, the cut's chunk through those over all its rows and through the rest from the cut on (Run::r0, a+ // view of its buffers), the chunks after it through every layer - each layer's experts read as without the+ // skip. One GPU (DS4_YOCO=1; 0: off); not with every row's logits (tests)+ const int yl = logits_rows > kWin ? -1 : yoco_layer();+ // (DS4_YOCO_KEEP=<rows>: fewer - DeepSeek's own serving replays the last 128 tokens through those layers+ // (DeepInfer's "SWA bounded replay"), which shifts the answers a little; at least one window)+ static const int keep_env = std::getenv("DS4_YOCO_KEEP") ? std::atoi(std::getenv("DS4_YOCO_KEEP")) : 0;+ const int keep = yl <= 0 ? 0+ : keep_env > 0 ? std::max(keep_env, g.win)+ : yoco_mode() == 2 ? g.win : (g.nl - yl) * (g.win - 1) + g.win;+ const int a = yl > 0 && n - keep >= kYocoMin ? n - keep : 0;+ const int nc = (n + chunk - 1) / chunk, per = (n + nc - 1) / nc;++ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "forward_chunks n %d nc %d per %d\n", n, nc, per);+ Run run;+ for (int x = 0; x < n; x += per) {+ run.p0.push_back(pos + x);+ run.T.push_back(std::min(per, n - x));+ run.seq.push_back(++win_seq);+ }+ if (a > 0) {+ run.yoco = yl;+ run.cut_k = a / per;+ run.cut_r = a - run.cut_k * per;+ }+ logits_all = logits_rows > kWin;+ if (logits_all) chunk_logits.resize((size_t) n * g.vocab);+ for (int t = 0; t < n; ++t) {+ toks.push_back(tk[t]);+ ctoks.push_back(ctok(pos + t, tk[t]));+ }+ plan_streams();+ if (snap_slot >= 0) { // feed_mark: the state before the run's last row, kept by its layers on the way+ run.snap_pos = pos + n - 1;+ for (int il = 0; il < g.nl; ++il) {+ Layer& l = L[(size_t) il];+ Dev& d = devs[(size_t) l.dev];+ if (l.snap_win == nullptr && !dalloc(d, l.snap_win, (size_t) g.win * win_rb(), err)) return false;+ if (l.c_kv != nullptr && l.snap_kv == nullptr &&+ (!dalloc(d, l.snap_kv, (size_t) l.ratio * g.hd, err) || !dalloc(d, l.snap_sc, (size_t) l.ratio * g.hd, err)))+ return false;+ }+ }+ const int T_last = run.T.back();+ n_logits = logits_all ? 0 : std::min(logits_rows, T_last);+ const double tl = now_ms();+ { // each chunk's end on the head device, for the progress reports+ cudaSetDevice(devs[(size_t) head_dev].id);+ while (chunk_done.size() < run.T.size()) {+ cudaEvent_t e = nullptr;+ CK(cudaEventCreateWithFlags(&e, cudaEventDisableTiming));+ chunk_done.push_back(e);+ }+ chunk_rec.reset(new std::atomic<int>[chunk_done.size()]);+ for (size_t i = 0; i < chunk_done.size(); ++i) chunk_rec[i].store(0);+ }+ prof_chunk = prof_on;+ const bool lm = lm_begin(run);+ struct LmEnd { // (every way out: its host memory freed once the device is done with it)+ Impl* p;+ bool on;+ ~LmEnd() {+ if (on) p->lm_end();+ }+ } lm_end_{this, lm};+ std::vector<std::string> errs(devs.size());+ halted = false;+ halt.store(false);+ {+ std::lock_guard<std::mutex> lk(halt_mu);+ halt_k = INT_MAX;+ halt_seen = -1;+ }+ bool halting = false;+ for (size_t di = 0; di < devs.size(); ++di)+ workers[di]->submit([this, di, &run, &errs] {+ std::string e;+ if (!run_dev_chunks((int) di, run, e)) errs[di] = e.empty() ? "a device's thread failed" : e;+ });+ size_t reported = 0;+ for (;;) {+ bool all = true;+ for (auto& w : workers) all = all && w->idle();+ if (!all && !halting && cancel && cancel()) { // a Stop: the devices end the run early+ halting = true;+ std::lock_guard<std::mutex> lk(halt_mu);+ halt_k = halt_seen + 1;+ halt.store(true, std::memory_order_release);+ }+ if (progress && lm) { // (layer-major: the share of the layers every chunk went through)+ const int nd = lm_layers.load(std::memory_order_acquire);+ if (nd > 0 && nd < g.nl) progress((int64_t) n * nd / g.nl);+ } else if (progress) { // chunks through every layer (the last layer's device past their event)+ while (reported < run.T.size() && chunk_rec[reported].load(std::memory_order_acquire) &&+ cudaEventQuery(chunk_done[reported]) == cudaSuccess)+ ++reported;+ if (reported > 0 && reported < run.T.size())+ progress(run.p0[reported - 1] + run.T[reported - 1] - run.p0[0]);+ }+ if (all) break;+ std::this_thread::sleep_for(std::chrono::milliseconds(50));+ }+ for (const std::string& e : errs) // (a device's failure first: a Stop's early end is not one)+ if (!e.empty() && e != kHalted) {+ err = e;+ return false;+ }+ for (const std::string& e : errs)+ if (e == kHalted) {+ halted = true;+ err = "stopped while reading the prompt";+ return false;+ }+ st.layer_ms += now_ms() - tl;+ eg_stats();+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ const double th = now_ms();+ CK(cudaStreamSynchronize(d.st));+ st.head_ms += now_ms() - th;+ if (prof_chunk) { // DS4_PROF: the run's GPU time by stage, every device's summed+ for (Dev& dv : devs) {+ cudaSetDevice(dv.id);+ CK(cudaStreamSynchronize(dv.st));+ }+ cudaSetDevice(d.id);+ prof_collect();+ std::fprintf(stderr,+ "ds4 prompt GPU time, %d tokens: attention %.0f ms, routing + shared expert %.0f ms, experts %.0f ms "+ "(with the waits for streamed ones), between layers %.0f ms\n",+ n, cprof_ms[1], cprof_ms[2], cprof_ms[3], cprof_ms[0]);+ { // the finer table (ms, % of the sum)+ static const char* nm_[16] = {"between layers", "attn rest (q_b, wo_a/b, hc_post)", "shared expert", "experts (incl waits)",+ "engram", "attn mHC (mix/split/norm)", "attn projections", "compressor + indexer",+ "q_b + rope", "sparse attention", "ffn mHC + router + sort", "", "", "", "", ""};+ double sum = 0;+ for (double v : cprof_ms) sum += v;+ for (int i = 0; i < 11; ++i)+ std::fprintf(stderr, "ds4 prof %-34s %9.1f ms %5.1f%%\n", nm_[i], cprof_ms[i], sum > 0 ? 100.0 * cprof_ms[i] / sum : 0.0);+ struct Agg {+ double ms = 0, flops = 0;+ long calls = 0, rows = 0, experts = 0;+ };+ std::map<std::pair<int, int>, Agg> agg; // (type, role 0 gate+up, 1 down)+ {+ std::lock_guard<std::mutex> lk(prof_mu);+ for (const EkMark& m : ek_marks) {+ float a = 0, b = 0;+ cudaSetDevice(m.dev);+ cudaEventElapsedTime(&a, m.e0, m.e1);+ cudaEventElapsedTime(&b, m.e1, m.e2);+ Agg& x = agg[{m.tg, 0}];+ x.ms += a;+ x.flops += 2.0 * 2.0 * m.nrows * (double) g.dim * (double) m.ff;+ x.calls++, x.rows += m.nrows, x.experts += m.nexp;+ Agg& y = agg[{m.td, 1}];+ y.ms += b;+ y.flops += 2.0 * m.nrows * (double) g.dim * (double) m.ff;+ y.calls++, y.rows += m.nrows, y.experts += m.nexp;+ ek_pool[m.dev].push_back(m.e0);+ ek_pool[m.dev].push_back(m.e1);+ ek_pool[m.dev].push_back(m.e2);+ }+ ek_marks.clear();+ }+ double tot = 0;+ for (auto& kv : agg) tot += kv.second.ms;+ for (auto& kv : agg)+ std::fprintf(stderr,+ "ds4 prof expert kernels type %-8s %-7s %9.1f ms calls %5ld experts %6ld (%.1f us each, %.1f a call) rows/expert %6.1f ~%5.1f TFLOPS\n",+ gg::iq::name(kv.first.first), kv.first.second ? "down" : "gate+up", kv.second.ms, kv.second.calls,+ kv.second.experts, kv.second.experts > 0 ? 1e3 * kv.second.ms / (double) kv.second.experts : 0.0,+ kv.second.calls > 0 ? (double) kv.second.experts / (double) kv.second.calls : 0.0,+ kv.second.experts > 0 ? (double) kv.second.rows / (double) kv.second.experts : 0.0,+ kv.second.ms > 0 ? kv.second.flops / kv.second.ms / 1e9 : 0.0);+ std::fprintf(stderr, "ds4 prof expert kernels total %.1f ms\n", tot);+ }+ for (double& v : cprof_ms) v = 0;+ prof_chunk = false;+ }+ if (o.verbose || std::getenv("DS4_FLOW_STATS") != nullptr) {+ const double ms = now_ms() - tl;+ for (size_t di = 0; di < chunks.size(); ++di) {+ const Chunk& c = *chunks[di];+ if (!c.flow) continue;+ const double db = (double) (c.io->direct_bytes() - c.io_d0) / 1e9;+ const double bb = (double) (c.io->buffered_bytes() - c.io_b0) / 1e9;+ const double fb = db + bb, rb = (double) c.it_ram * (double) L[0].blob / 1e9; // (a member: its parts)+ char lcs[160] = "";+ if (c.lm)+ std::snprintf(lcs, sizeof lcs, ", %llu again from the layer cache (layer-major, %d chunks, %.1f GB "+ "of RAM)", (unsigned long long) c.it_lc, (int) run.T.size(), (double) c.lc_bytes / 1e9);+ char dbgs[200] = "";+ if (pm.on)+ std::snprintf(dbgs, sizeof dbgs,+ "; io threads busy %.0f ms, copy_ahead staging %.0f ms, DMA issues %.0f ms, slot waits %.0f ms",+ c.io->busy_ms() - c.io_busy0, c.dbg_stage_ms, c.dbg_dma_ms, c.dbg_evt_ms);+ std::fprintf(stderr,+ "ds4 prompt stream, CUDA%d: %d tokens in %.0f ms; %llu experts streamed: %llu from the files "+ "(%.1f GB, %.2f GB/s: %.1f GB from the disk unbuffered, %.1f GB through the page cache), %llu "+ "from RAM (%.1f GB)%s; %llu likely ones queued early, %llu of them unpicked; the compute waited "+ "%.0f ms for reads%s\n",+ devs[di].id, n, ms, (unsigned long long) (c.it_file + c.it_ram + c.it_lc),+ (unsigned long long) c.it_file, fb, fb / std::max(1e-3, ms / 1000.0), db, bb,+ (unsigned long long) c.it_ram, rb, lcs, (unsigned long long) c.it_spec,+ (unsigned long long) c.it_spec_unused, c.wait_ms, dbgs);+ }+ }+ if (pm.on && run.T.size() == 1) { // (the picks' spread for pick_path: the CPU-lane experts this run picked)+ bool tp = false;+ for (const Layer& l : L) tp = tp || l.nm > 1;+ double picked = 0;+ for (const auto& cp : chunks)+ if (cp != nullptr && cp->flow) picked += (double) (cp->it_file + cp->it_ram) - (double) cp->it_spec_unused;+ if (!tp) pm_learn(n, picked);+ }+ if (logits_all) n_logits = (T_last - 1) % kWin + 1; // (sample() takes the last row of d.logits)+ pos += n;+ st.tokens += n;+ cache_tick(n);+ if (lm_end_.on) { // (the layer cache's RAM back before the store and the swaps want it)+ lm_end();+ lm_end_.on = false;+ }+ cache_settle();+ const int nr = std::min(T_last, tap_cap);+ if (!seed_drafter(pos - nr, nr, 0, err)) return false;+ if (snap_slot >= 0) { // checkpoint `snap_slot` from what the layers kept (the drafter's blocks: as they are -+ // the next feed seeds them again from its own rows, past this position)+ Check& ckp = ck[snap_slot & 1];+ ckp.pos = run.snap_pos;+ ckp.win.resize(L.size());+ ckp.comp.resize(L.size() * 2);+ for (size_t i = 0; i < L.size(); ++i) {+ Layer& l = L[i];+ Dev& dv = devs[(size_t) l.dev];+ cudaSetDevice(dv.id);+ const bool kept = !l.mtp && l.snap_win != nullptr;+ ckp.win[i].resize((size_t) g.win * win_rb());+ CK(cudaMemcpyAsync(ckp.win[i].data(), kept ? l.snap_win : l.win, ckp.win[i].size(),+ cudaMemcpyDeviceToHost, dv.st));+ if (l.c_kv) {+ ckp.comp[2 * i].resize((size_t) l.ratio * g.hd);+ ckp.comp[2 * i + 1].resize((size_t) l.ratio * g.hd);+ CK(cudaMemcpyAsync(ckp.comp[2 * i].data(), kept && l.snap_kv ? l.snap_kv : l.c_kv,+ ckp.comp[2 * i].size() * 4, cudaMemcpyDeviceToHost, dv.st));+ CK(cudaMemcpyAsync(ckp.comp[2 * i + 1].data(), kept && l.snap_sc ? l.snap_sc : l.c_score,+ ckp.comp[2 * i + 1].size() * 4, cudaMemcpyDeviceToHost, dv.st));+ }+ }+ for (Dev& dv : devs) {+ cudaSetDevice(dv.id);+ CK(cudaStreamSynchronize(dv.st));+ }+ }+ return true;+ }++ // the window's plan, in window order (the indexer's and the candidate source's sizes carry from window to window)+ void plan_window(int64_t p0, int T, WinPlan& wp) {+ wp.par = par;+ wp.async = par || (!sync_decode && T <= async_max_t);+ wp.vl = 0;+ for (int t = 0; t < T && !imgs.empty(); ++t)+ if (in_image(p0 + t)) wp.vl |= 1u << t;+ for (int il = 0; il < g.nl; ++il) {+ const Layer& l = L[(size_t) il];+ if (l.ratio > 0) {+ const int R = l.ratio;+ int n[kWin], k[kWin], clen_max = 0;+ for (int t = 0; t < T; ++t) {+ n[t] = (int) ((p0 + t + 1) / R);+ k[t] = std::min(g.ix_topk, n[t]);+ clen_max = std::max(clen_max, n[t]);+ }+ if (l.ix_src) {+ for (int t = 0; t < T; ++t) wp.cand[il][t] = cur_cand_n[t];+ if (clen_max > 0 && l.cand_src)+ for (int t = 0; t < T; ++t) cur_cand_n[t] = std::min(g.cand_topk, (n[t] + g.cand_blk - 1) / g.cand_blk);+ for (int t = 0; t < T; ++t) cur_topk_n[t] = k[t];+ }+ for (int t = 0; t < T; ++t) wp.topk[il][t] = cur_topk_n[t];+ }+ if (wp.async) {+ wp.seq[il] = ++seq_issued;+ layer_of[wp.seq[il] % kRing].store(il, std::memory_order_release);+ }+ }+ }++ int head_nm() const { return (int) groups[(size_t) devs[(size_t) head_dev].grp].mem.size(); }+ // the head's columns on member mi of its group of nm: an EXL3 head's split128 of the vocabulary (mat's), a GGUF's+ // BF16 / Q8_0 head's even shares (mat_bf16's) - head_share's, its gather's and the markov steps' rows+ void head_cols(int nm, int mi, int& v0, int& vn) const {+ const Tensor* tw = pk.get("head.weight");+ if (tw != nullptr && (tw->dtype == "BF16" || tw->dtype == "Q8_0")) {+ vn = g.vocab / nm;+ v0 = mi * vn;+ } else+ split128(g.vocab, nm, mi, v0, vn);+ }+ // the head over `rows` (<= 16) normed rows xn on member mi of the head group (device d): its share of each row's+ // logits (columns [v0, v0 + vn), head_cols); the lead gathers the others' into d.logits (rows V apart) - two+ // slots by seq (Vmax apart, the widest share's), a member reusing one once the lead acknowledged it. gather+ // false: each member keeps its share (the drafter's markov steps over the group, mk_split)+ bool head_share(Dev& d, int mi, const float* xn, int rows, std::string& err, bool gather = true) {+ const int V = g.vocab, D = g.dim, nm = head_nm();+ int v0 = 0, vn = 0, Vmax = 0, z = 0;+ head_cols(nm, mi, v0, vn);+ head_cols(nm, 0, z, Vmax);+ Group& G = groups[(size_t) d.grp];+ float* out = mi == 0 ? d.logits : d.logits_part;+ const size_t ld = mi == 0 ? (size_t) V : (size_t) vn;+ cudaSetDevice(d.id);+ // (an EXL3 head: as many activations at a time as its split-K partials fit the scratch - 8 for a shared head, 4+ // for a whole one, each pass reading the whole share; a GGUF's: 8, so that every window the engine can build (rows+ // <= kWin = 8) walks the head once instead of twice - measured on the drafter's fixed T=5: 4.55 -> 2.83 ms, its+ // round 14.49 -> 13.46; the verify window's head 3.48 -> 2.68 ms, the same logits. DS4_HEAD_PER overrides.)- const int per = d.head.fmt != 0 ? (std::getenv("DS4_HEAD_PER") ? std::atoi(std::getenv("DS4_HEAD_PER")) : strata::ds4::head_per_default(defaults_026()))++ const int per = d.head.fmt != 0 ? (std::getenv("DS4_HEAD_PER") ? std::atoi(std::getenv("DS4_HEAD_PER")) :++#if defined(STRATA_METAL)++ 4 // (8 columns a pass on Metal moved the KL by +0.002: the mvn8 kernel sums in another order)++#else++ strata::ds4::head_per_default(defaults_026())++#endif++ )+ : std::max(1, x3::mv_cols(d.head));+ for (int c = 0; c < rows; c += per)+ if (!mv(d, {job(d.head, xn + (size_t) c * D, out + (size_t) c * ld, std::min(per, rows - c), D, ld)}, err))+ return false;+ if (nm == 1 || !gather) return true;+ const unsigned seq = ++d.head_seq;+ const int slot = (int) (seq % 2);+ auto stage = [&](int m) { return G.lg_stage + ((size_t) slot * (nm - 1) + (m - 1)) * kWin * Vmax; };+ auto flag = [&](int m) { return G.lg_flag + ((size_t) slot * (nm - 1) + (m - 1)) * 32; };+ if (mi > 0) {+ if (seq > 2) wait_flags(G.lg_ack + (size_t) slot * 32, 1, 32, seq - 2, d.st);+ Xfer x;+ x.n = 1;+ x.src[0] = d.logits_part;+ x.dst[0] = stage(mi);+ x.bytes[0] = (size_t) rows * vn * 4;+ xfer_put(x, flag(mi), seq, d.xcnt, d.st);+ return true;+ }+ for (int m = 1; m < nm; ++m) {+ int m0 = 0, mn = 0;+ head_cols(nm, m, m0, mn);+ Xfer x;+ x.n = 1;+ x.src[0] = stage(m);+ x.dst[0] = d.logits_part;+ x.bytes[0] = (size_t) rows * mn * 4;+ xfer_get(x, flag(m), seq, d.st);+ CK(cudaMemcpy2DAsync(d.logits + (size_t) m0, (size_t) V * 4, d.logits_part, (size_t) mn * 4,+ (size_t) mn * 4, (size_t) rows, cudaMemcpyDeviceToDevice, d.st));+ }+ set_flag(G.lg_ack + (size_t) slot * 32, seq, d.st);+ return true;+ }++ // the head's rows (the last n_logits of the window) on head group member di (its h / pre: the group's)+ bool head_rows_dev(int di, int T, std::string& err) {+ Dev& d = devs[(size_t) di];+ cudaSetDevice(d.id);+ const int D = g.dim, H = g.hc;+ if (n_logits <= 0) return true;+ const int t0 = T - n_logits;+ hc_pre_norm(d.h + (size_t) t0 * H * D, d.pre + (size_t) t0 * 8, d.norm_w, D, H, n_logits, g.eps, d.x, d.st);+ return head_share(d, d.mi, d.x, n_logits, err);+ }++ // the DSpark drafter's taps: the streams' mean entering target layer il, on device d (after the last seeding read+ // h_taps)+ bool tap(Dev& d, int il, int T, std::string& err) {+ const auto it = std::find(g.d_targets.begin(), g.d_targets.end(), il);+ if (g.n_mtp == 0 || it == g.d_targets.end()) return true;+ const size_t j = (size_t) (it - g.d_targets.begin()), nt = g.d_targets.size();+ const int D = g.dim;+ cudaSetDevice(d.id);+ hc_mean(d.h, T, g.hc, D, d.tap, D, d.st);+ CK(cudaStreamWaitEvent(d.st, ev_seed, 0));+ CK(cudaMemcpy2DAsync(h_taps + j * D, nt * D * 4, d.tap, (size_t) D * 4, (size_t) D * 4, (size_t) T,+ cudaMemcpyDeviceToHost, d.st));+ return true;+ }++ bool forward(const int* tk, int T, int logits_rows, std::string& err, int seed_rows = -1) {++#if defined(STRATA_METAL)++ struct BusyAll { // DS4_BUSY_ALL: the GPU is held busy for the whole step (the host gaps between its bursts too)++ bool on;++ BusyAll() : on(std::getenv("DS4_BUSY_ALL") != nullptr) { if (on) ds4_busy_begin(); }++ ~BusyAll() { if (on) ds4_busy_end(); }++ } busy_all;++#endif+ if (T < 1 || T > kWin) {+ err = "a window of " + std::to_string(T) + " tokens";+ return false;+ }+ if (pos + T > o.max_context) {+ err = "the context is full";+ return false;+ }+ for (int t = 0; t < T; ++t)+ if (tk[t] < 0 || tk[t] >= g.vocab) {+ err = "token " + std::to_string(tk[t]) + " out of range";+ return false;+ }+ const int64_t p0 = pos;+ slot_held = false; // (the state grows now: what slot_save took is no longer the conversation)+ for (int t = 0; t < T; ++t) {+ toks.push_back(tk[t]);+ ctoks.push_back(ctok(pos + t, tk[t]));+ }+ // the Engram rows asked for now (the first Engram layer's first), collected right before their layer: what is+ // left of their reads by then is the window's Engram time+ const double te = now_ms();+ const int64_t eg_w0 = eg_wait_ns.load();+ for (int li = 0; li < pk.eg_layers; ++li) {+ EgBatch& b = *eg_win[(size_t) li];+ if (b.live) eg_finish(b); // (a window that failed midway: its reads end before h_eg is reused)+ eg_issue(p0, T, li, nullptr, b);+ }+ st.engram_ms += now_ms() - te;+ const int D = g.dim;+ for (int t = 0; t < T; ++t) // (h_emb is written again by the next window, after this one's last sync)+ std::memcpy(h_emb + (size_t) t * D, emb_row(p0 + t, tk[t]), (size_t) D * 2);+ plan_window(p0, T, wplan);+ prof_window = prof_on && T == 1;+ n_logits = std::min(logits_rows, T);+ const double tl = now_ms();+ if (par) {+ if (!forward_par(p0, T, err)) return false;+ } else if (!forward_serial(p0, T, err)) return false;+ st.layer_ms += now_ms() - tl;+ for (auto& b : eg_win)+ if (b->live) eg_finish(*b);+ st.engram_ms += (double) (eg_wait_ns.load() - eg_w0) / 1e6;+ eg_stats();+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ const double th = now_ms();+ CK(cudaStreamSynchronize(d.st));+ st.head_ms += now_ms() - th;+ prof_collect();+ pos += T;+ st.tokens += T;+ cache_tick(T);+ if (seed_rows != 0 && !seed_drafter(p0, seed_rows < 0 ? T : seed_rows, 0, err)) return false;+ return true;+ }++ // one thread: the layers device after device, the hand-offs through pinned memory and events++ void dbg_nf(const char* tag, int il, const float* dev, size_t n) { // DBG: non-finite count of a device buffer++ ds4_metal_flush();++ const float* hp = (const float*) strata::gpu::host_view(dev);++ size_t nf = 0, first = 0; float mx = 0.0f;++ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) { if (!nf) first = i; ++nf; } else if (std::fabs(v) > mx) mx = std::fabs(v); }++ std::fprintf(stderr, "DBG nf il=%d %s n=%zu nonfinite=%zu first=%zu max=%.4g\n", il, tag, n, nf, first, mx);++ }++ void dbg_h(const char* tag, Dev& d, int T) { // DBG: max |h| of the window's residual stream, after the queue drains++ ds4_metal_flush();++ const float* hp = (const float*) strata::gpu::host_view(d.h);++ float mx = 0.0f; size_t nf = 0;++ const size_t n = (size_t) T * g.hc * g.dim;++ for (size_t i = 0; i < n; ++i) { const float v = hp[i]; if (!(v == v)) ++nf; else if (std::fabs(v) > mx) mx = std::fabs(v); }++ std::fprintf(stderr, "DBG hmax %s T=%d max=%.4g nonfinite=%zu h0=%g\n", tag, T, mx, nf, hp[0]);++ }+ bool forward_serial(int64_t p0, int T, std::string& err) {+ const int D = g.dim, H = g.hc;+ int cur = L[0].dev;++ if (std::getenv("DS4_SYNC_WIN")) cudaDeviceSynchronize();+ {+ Dev& d = devs[(size_t) cur];+ cudaSetDevice(d.id);- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, d.st));++ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));+ embed(d.emb_rows, T, d.h, D, H, d.st);++ if (std::getenv("DS4_DBG_EMB")) {++ ds4_metal_flush();++ float *hp = d.h;++ float hh[4] = {0,0,0,0};++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);++ }+ init_pre(d.pre, T, d.st);++ if (std::getenv("DS4_DBG_EMB")) dbg_h("after init_pre", d, T);+ }+ std::vector<float> dump;+ if (dump_h != nullptr) dump.resize((size_t) T * g.nl * H * D);+ for (int il = 0; il < g.nl; ++il) {+ Layer& l = L[(size_t) il];+ if (l.dev != cur) {+ Dev& a = devs[(size_t) cur];+ Dev& b = devs[(size_t) l.dev];+ cudaSetDevice(a.id);+ const size_t nh = (size_t) T * H * D * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,+ nc = (size_t) T * cand_stride() * 4;+ uint8_t* bb = a.bounce;+ CK(cudaMemcpyAsync(bb, a.h, nh, cudaMemcpyDeviceToHost, a.st));+ CK(cudaMemcpyAsync(bb + nh, a.pre, np, cudaMemcpyDeviceToHost, a.st));+ CK(cudaMemcpyAsync(bb + nh + np, a.topk, nk, cudaMemcpyDeviceToHost, a.st));+ CK(cudaMemcpyAsync(bb + nh + np + nk, a.cand, nc, cudaMemcpyDeviceToHost, a.st));+ CK(cudaEventRecord(a.ev_out, a.st));+ cudaSetDevice(b.id);+ CK(cudaStreamWaitEvent(b.st, a.ev_out, 0));+ CK(cudaMemcpyAsync(b.h, bb, nh, cudaMemcpyHostToDevice, b.st));+ CK(cudaMemcpyAsync(b.pre, bb + nh, np, cudaMemcpyHostToDevice, b.st));+ CK(cudaMemcpyAsync(b.topk, bb + nh + np, nk, cudaMemcpyHostToDevice, b.st));+ CK(cudaMemcpyAsync(b.cand, bb + nh + np + nk, nc, cudaMemcpyHostToDevice, b.st));+ cur = l.dev;+ }+ if (!tap(devs[(size_t) l.dev], il, T, err)) return false;++ if (il == 0 && std::getenv("DS4_DBG_EMB")) dbg_h("after tap", devs[(size_t) l.dev], T);+ if (!layer(l, 0, p0, T, wplan, err)) return false;+ if (dump_h != nullptr) { // DS4_DUMP_H: every layer's residual stream, for tools/ds4/compare.py --layers+ Dev& dd = devs[(size_t) l.dev];+ std::vector<float> hv((size_t) T * H * D);+ cudaSetDevice(dd.id);+ CK(cudaMemcpyAsync(hv.data(), dd.h, hv.size() * 4, cudaMemcpyDeviceToHost, dd.st));+ CK(cudaStreamSynchronize(dd.st));+ for (int t = 0; t < T; ++t)+ std::memcpy(dump.data() + ((size_t) t * g.nl + l.il) * H * D, hv.data() + (size_t) t * H * D,+ (size_t) H * D * 4);+ }+ }+ if (dump_h != nullptr) std::fwrite(dump.data(), 4, dump.size(), dump_h);+ if (cur != head_dev) {+ err = "the head is not on the last layer's GPU";+ return false;+ }+ return head_rows_dev(head_dev, T, err);+ }++ // threads: each device's thread enqueues its layers (a group's members each their share) at once, the+ // devices waiting on each other through host-mapped flags (the window's seq)+ bool forward_par(int64_t p0, int T, std::string& err) {+ wplan.win = ++win_seq;+ std::vector<std::string> errs(devs.size());+ for (size_t di = 0; di < devs.size(); ++di)+ workers[di]->submit([this, di, p0, T, &errs] {+ std::string e;+ if (!run_dev((int) di, p0, T, wplan, e)) errs[di] = e.empty() ? "a device's thread failed" : e;+ });+ for (auto& w : workers) w->wait();+ for (const std::string& e : errs)+ if (!e.empty()) {+ err = e;+ return false;+ }+ return true;+ }++ // the hand-off between groups: h | pre | topk | cand of device d <-> the host-mapped buffer+ Xfer hand_xfer(uint8_t* host, Dev& d, int T, bool put) const {+ const size_t nh = (size_t) T * g.hc * g.dim * 4, np = (size_t) T * 8 * 4, nk = (size_t) T * topk_stride() * 4,+ nc = (size_t) T * cand_stride() * 4;+ void* dv[4] = {d.h, d.pre, d.topk, d.cand};+ uint8_t* hv[4] = {host, host + nh, host + nh + np, host + nh + np + nk};+ const size_t nb[4] = {nh, np, nk, nc};+ Xfer x;+ x.n = 4;+ for (int i = 0; i < 4; ++i) {+ x.src[i] = put ? (const void*) dv[i] : (const void*) hv[i];+ x.dst[i] = put ? (void*) hv[i] : dv[i];+ x.bytes[i] = nb[i];+ }+ return x;+ }++ bool run_dev(int di, int64_t p0, int T, const WinPlan& wp, std::string& err) {+ Dev& d = devs[(size_t) di];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ const int D = g.dim, H = g.hc;+ bool started = false;+ std::vector<char> got(srcs.size(), 0);+ for (int il = 0; il < g.nl; ++il) {+ Layer& L0 = L[(size_t) il];+ int mi = -1;+ for (int m = 0; m < L0.nm; ++m)+ if (member(L0, m).dev == di) mi = m;+ if (mi < 0) continue;+ if (!started) { // the window's input: the embedding rows, or the previous group's hand-off+ started = true;+ if (il == 0) {- CK(cudaMemcpyAsync(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice, s));++ CK(cudaMemcpy(d.emb_rows, h_emb, (size_t) T * D * 2, cudaMemcpyHostToDevice));+ embed(d.emb_rows, T, d.h, D, H, s);++ if (std::getenv("DS4_DBG_EMB")) {++ ds4_metal_flush();++ float *hp = d.h;++ float hh[4] = {0,0,0,0};++ cudaMemcpy(hh, hp, sizeof(hh), cudaMemcpyDeviceToHost);++ std::fprintf(stderr, "DBG at embed site %d: h[0..3] = %g %g %g %g\n", __LINE__, hh[0], hh[1], hh[2], hh[3]);++ }+ init_pre(d.pre, T, s);+ } else {+ const Dev& a = devs[(size_t) L[(size_t) il - 1].dev];+ xfer_get(hand_xfer(a.hand, d, T, false), a.hand_flag, wp.win, s);+ }+ }+ // the compressed KV / index keys another group's source layer wrote this window+ for (int sidx : {L0.src, L0.ix_src ? L0.ik_src : -1}) {+ if (sidx < 0 || got[(size_t) sidx]) continue;+ got[(size_t) sidx] = 1;+ Src& sr = srcs[(size_t) sidx];+ if (L[(size_t) sr.layer].grp == L0.grp) continue;+ const int64_t g_first = p0 / sr.ratio;+ const int ng = (int) std::max<int64_t>(0, (p0 + T) / sr.ratio - g_first);+ if (ng == 0) continue;+ Xfer x;+ if (sr.ckv[(size_t) di] != nullptr) {+ x.src[x.n] = sr.x_stage;+ x.dst[x.n] = sr.ckv[(size_t) di] + (size_t) g_first * ckv_rb();+ x.bytes[x.n++] = (size_t) ng * ckv_rb();+ }+ if (sr.ik[(size_t) di] != nullptr) {+ x.src[x.n] = sr.x_stage + x_cap() * ckv_rb();+ x.dst[x.n] = sr.ik[(size_t) di] + (size_t) g_first * ik_rb();+ x.bytes[x.n++] = (size_t) ng * ik_rb();+ }+ if (x.n > 0) xfer_get(x, sr.x_flag, wp.win, s);+ }+ if (mi == 0 && !tap(d, il, T, err)) return false;+ if (!layer(L0, mi, p0, T, wp, err)) return false;+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, il, p0, T);+ const bool last = il + 1 == g.nl || L[(size_t) il + 1].grp != L0.grp;+ if (last && mi == 0 && il + 1 < g.nl) xfer_put(hand_xfer(d.hand, d, T, true), d.hand_flag, wp.win, d.xcnt, s);+ }+ if (d.grp == devs[(size_t) head_dev].grp) return head_rows_dev(di, T, err);+ return true;+ }++ // a source's new cache rows (src on device `from`, bytes of them) to the same place of its other devices' copies,+ // through pinned memory: the copy out on the source's stream, each copy in on its device's stream after it+ bool replicate(std::vector<uint8_t*>& copies, uint8_t* stage, cudaEvent_t ev, const uint8_t* src, size_t at,+ size_t bytes, int from, std::string& err) {+ bool any = false;+ for (size_t od = 0; od < devs.size(); ++od) any = any || (copies[od] != nullptr && (int) od != from);+ if (!any) return true;+ Dev& d = devs[(size_t) from];+ CK(cudaMemcpyAsync(stage, src, bytes, cudaMemcpyDeviceToHost, d.st));+ CK(cudaEventRecord(ev, d.st));+ for (size_t od = 0; od < devs.size(); ++od) {+ if (copies[od] == nullptr || (int) od == from) continue;+ Dev& o2 = devs[od];+ cudaSetDevice(o2.id);+ CK(cudaStreamWaitEvent(o2.st, ev, 0));+ CK(cudaMemcpyAsync(copies[od] + at, stage, bytes, cudaMemcpyHostToDevice, o2.st));+ }+ cudaSetDevice(d.id);+ return true;+ }++ // layer L0 for the window, as group member mi (its share of the heads, of the output projection and of the FFNs;+ // the partial sums all-reduced across the group)+ bool layer(Layer& L0, int mi, int64_t p0, int T, const WinPlan& wp, std::string& err) {+ Layer& l = member(L0, mi);+ Dev& d = devs[(size_t) l.dev];+ cudaSetDevice(d.id);+ cudaStream_t s = d.st;+ const int nm = L0.nm;+ const int D = g.dim, H = g.hc, HD = g.hd, NHL = l.ahn, HQ = NHL * HD, OG = l.ogn, FF = l.ffn;+ const ArGroup& ag = groups[(size_t) L0.grp].ar;+ const float* freqs = l.ratio ? d.freqs_comp : d.freqs_win;++ if (std::getenv("DS4_DBG_SLOTS") && l.il < 2 && l.lay.fmt == gg::kFmtMXFP4) { // DBG: a resident expert's bytes against its file's++ ds4_metal_flush();++ int shown = 0;++ for (int e = 0; e < l.n_exp && shown < 3; ++e) {++ if (l.slot[(size_t) e] < 0) continue;++ std::vector<uint8_t> want(l.blob, 0);++ read_mx_file(l, e, want.data());++ const uint8_t* have = (const uint8_t*) strata::gpu::host_view(l.vram_base + (size_t) l.slot[(size_t) e] * unit(l));++ size_t bad = 0, first = 0;++ for (size_t i = 0; i < l.blob; ++i)++ if (want[i] != have[i]) { if (!bad) first = i; ++bad; }++ std::fprintf(stderr, "DBG slot il=%d e=%d slot=%d blob=%zu mismatching bytes=%zu first=%zu\n", l.il, e, l.slot[(size_t) e], l.blob, bad, first);++ ++shown;++ }++ }++ if (std::getenv("DS4_TRACE_DISP")) std::fprintf(stderr, "TRACE layer %d p0=%lld T=%d\n", l.il, (long long) p0, T);++ if (std::getenv("DS4_DBG_H") && mi == 0 && (l.il < 2 || std::getenv("DS4_DBG_ALL"))) { // DBG++ std::vector<float> hv((size_t) T * H * D);++ cudaSetDevice(d.id);++ ds4_metal_flush();++ {++ const float* hvp = (const float*) strata::gpu::host_view(d.h);++ if (hvp) std::memcpy(hv.data(), hvp, hv.size() * 4);++ }++ int nf = 0, nzv = 0;++ float mx = 0.0f;++ size_t firstnan = 0, lastnan = 0;++ for (size_t i = 0; i < hv.size(); ++i) {++ const float v = hv[i];++ if (!(v == v)) { if (nf == 0) firstnan = i; lastnan = i; ++nf; continue; }++ if (v != 0.0f) ++nzv;++ if (std::fabs(v) > mx) mx = std::fabs(v);++ }++ std::fprintf(stderr, "DBG h: il=%d dev=%d h=%p p0=%lld T=%d p0tok=%d zeros=%zu ninf=%d max=%.4g first=%.5g %.5g nan_range=%zu..%zu\n",++ l.il, (int) l.dev, (void*) d.h, (long long) p0, T, toks.empty() ? -1 : toks[(size_t) p0],++ hv.size() - (size_t) nzv - (size_t) nf, nf, mx, hv[0], hv.size() > 1 ? hv[1] : 0.0f, firstnan, lastnan);++ }+ prof_mark(d, 0);+ // Engram+ if (l.eg >= 0) {+ const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;+ const size_t per = (size_t) cols * (ehd + ehd / 32);+ eg_finish(*eg_win[(size_t) l.eg]); // (asked for when the window began)+ uint8_t* src = h_eg + (size_t) l.eg * eg_cap * per;+ for (int t = 0; t < T; ++t) // [token][rows | scales] -> [all rows][all scales]+ CK(cudaMemcpyAsync(d.eg_rows + (size_t) t * cols * ehd, src + (size_t) t * per, (size_t) cols * ehd,+ cudaMemcpyHostToDevice, s));+ uint8_t* sc = d.eg_rows + (size_t) T * cols * ehd;+ for (int t = 0; t < T; ++t)+ CK(cudaMemcpyAsync(sc + (size_t) t * cols * (ehd / 32), src + (size_t) t * per + (size_t) cols * ehd,+ (size_t) cols * (ehd / 32), cudaMemcpyHostToDevice, s));+ engram_dequant(d.eg_rows, sc, T * cols, ehd, d.eg_x, s, (int) eg_fmt[(size_t) l.eg]);++ dump_sub(d, "eg_rows", l.il, p0, T, d.eg_rows, (size_t) T * cols * ehd);++ dump_sub(d, "eg_x", l.il, p0, T, d.eg_x, (size_t) T * cols * ehd * 4);++ if (std::getenv("DS4_DBG_EG")) {++ dbg_nf("eg_x", l.il, d.eg_x, (size_t) T * cols * ehd);++ const float* xv = (const float*) strata::gpu::host_view(d.eg_x);++ for (size_t i = 0; i < (size_t) T * cols * ehd; ++i)++ if (!std::isfinite(xv[i])) {++ const size_t t = i / ((size_t) cols * ehd), c = (i / ehd) % cols;++ const uint8_t* dv = (const uint8_t*) strata::gpu::host_view(d.eg_rows) + (t * cols + c) * ehd;++ const uint8_t* hv = src + t * per + c * ehd;++ std::fprintf(stderr, "DBG eg bad x: t=%zu c=%zu e=%zu | dev/host row bytes:", t, c, i % ehd);++ for (int b = 0; b < 80; ++b) std::fprintf(stderr, "%s%02x/%02x", b % 18 == 0 ? " |" : " ", dv[b], hv[b]);++ std::fprintf(stderr, "\n");++ break;++ }++ }+ if (!mv(d, {job(l.eg_wkv, d.eg_x, d.eg_kv, T, (size_t) cols * ehd, (size_t) (H + 1) * D)}, err)) return false;++ if (std::getenv("DS4_DBG_EG")) dbg_nf("eg_kv", l.il, d.eg_kv, (size_t) T * (H + 1) * D);++ dump_sub(d, "eg_kv", l.il, p0, T, d.eg_kv, (size_t) T * (H + 1) * D * 4);++ dump_sub(d, "eg_h0", l.il, p0, T, d.h, (size_t) T * H * D * 4);++ dump_sub(d, "eg_w", l.il, p0, 1, l.eg_w, (size_t) H * D * 4);+ engram_gate(d.h, d.eg_kv, l.eg_w, H, D, T, g.eps, s);++ dump_sub(d, "eg_h", l.il, p0, T, d.h, (size_t) T * H * D * 4);++ if (std::getenv("DS4_DBG_EG")) dbg_nf("h_after_gate", l.il, d.h, (size_t) T * H * D);+ }++ dump_sub(d, "h_in", l.il, p0, T, d.h, (size_t) T * H * D * 4);++ dump_sub(d, "ring_in", l.il, p0, T, l.win, (size_t) g.win * kv_row_bytes(kKvFp8, HD));+ // attention++#if defined(STRATA_METAL)++ hc_mix_split(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, l.hca_scale, l.hca_base, H,++ g.sinkhorn, g.hc_eps, d.a_pre, d.a_post, d.a_comb, s);++#else+ hc_mix_split(d.h, l.hca_fn, H * D, (2 + H) * H, T, g.eps, d.mix, d.hccnt, l.hca_scale, l.hca_base, H, g.sinkhorn, g.hc_eps,+ d.a_pre, d.a_post, d.a_comb, s);++#endif++ dump_sub(d, "a_mix", l.il, p0, T, d.mix, (size_t) T * 32 * 4);++ dump_sub(d, "a_pre", l.il, p0, T, d.a_pre, (size_t) T * 8 * 4);++ dump_sub(d, "a_post", l.il, p0, T, d.a_post, (size_t) T * 8 * 4);++ dump_sub(d, "a_comb", l.il, p0, T, d.a_comb, (size_t) T * 64 * 4);+ hc_pre_norm(d.h, d.pre, l.attn_norm, D, H, T, g.eps, d.x, s);++ dump_sub(d, "pre_in", l.il, p0, T, d.pre, (size_t) T * 8 * 4);++ dump_sub(d, "attn_x", l.il, p0, T, d.x, (size_t) T * D * 4);+ {+ x3::MvJob j[4];+ int nj = 0;+ j[nj++] = job(l.wq_a, d.x, d.qr, T, D, g.q_lora);+ j[nj++] = job(l.wkv, d.x, d.kvr, T, D, HD);+ if (l.kv_src) j[nj++] = job(l.c_wkv, d.x, d.ckr, T, D, HD);+ if (l.kv_src && l.ratio > 1) j[nj++] = job(l.c_wgate, d.x, d.cgr, T, D, HD);+ if (!mv_n(d, j, nj, err)) return false;+ }++ dump_sub(d, "qr_raw", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);++ dump_sub(d, "kvr_raw", l.il, p0, T, d.kvr, (size_t) T * HD * 4);+ Finish fq;+ fq.w = l.q_norm;+ fq.eps = g.eps;+ finish(d.qr, T, g.q_lora, g.q_lora, fq, s);++ dump_sub(d, "qr", l.il, p0, T, d.qr, (size_t) T * g.q_lora * 4);+ if (NHL > 0) {+ if (!mv(d, {job(l.wq_b, d.qr, d.q, T, g.q_lora, HQ)}, err)) return false;+ Finish fr;+ fr.rd = g.rd;+ fr.pos = (int) p0;+ fr.vpp = NHL;+ fr.freqs = freqs;+ finish(d.q, T * NHL, HD, HD, fr, s);+ }++ if (NHL > 0) dump_sub(d, "q", l.il, p0, T, d.q, (size_t) T * HQ * 4);+ // the window rows: the ring's (positions p0 - win ..) then this window's, each query its own 128. A DSpark+ // block's ring holds the trunk's main KV (positions <= p0 - 1, seeded); its block of drafts attends to all of+ // that ring and to the whole block, and leaves the ring as it was+ ring_to_ext(l.win, d.ext, l.mtp ? (int64_t) g.win : p0, g.win, HD, s); // (a block's ring: its rows as they lie)+ Finish fk;+ fk.w = l.kv_norm;+ fk.eps = g.eps;+ fk.rd = g.rd;+ fk.pos = (int) p0;+ fk.freqs = freqs;+ fk.quant = 1;+ fk.dst = d.ext + (size_t) g.win * HD;+ fk.dst_stride = HD;+ finish(d.kvr, T, HD, HD, fk, s);+ if (!l.mtp) ext_to_ring(d.ext, l.win, p0, g.win, T, HD, s, spec_win ? l.ring_bak : nullptr);++ dump_sub(d, "kv_win", l.il, p0, T, d.ext + (size_t) g.win * HD, (size_t) T * HD * 4);++ dump_sub(d, "ext", l.il, p0, T, d.ext, ((size_t) g.win + T) * HD * 2); // (BF16 rows: the ring's, then this window's)+ AttnRows ar;+ for (int t = 0; t < T; ++t) {+ if (l.mtp) {+ ar.win0[t] = 0;+ ar.nwin[t] = (int) std::min<int64_t>(g.win, p0);+ ar.nsel[t] = T;+ continue;+ }+ const int first = (int) std::max<int64_t>(g.win - p0, t + 1); // ext row of position max(0, p - win + 1)+ ar.win0[t] = first;+ ar.nwin[t] = g.win + t - first + 1;+ ar.nsel[t] = 0;+ }+ const void* ckv = nullptr;+ int ckv_fmt = kKvFp4;+ if (l.ratio > 0) {+ Src& sr = srcs[(size_t) l.src];+ const int R = l.ratio;+ // the groups this window completes: g_first .. g_first + ng - 1+ const int64_t g_first = p0 / R; // the first group whose last position is >= p0+ const int64_t g_end = (p0 + T) / R; // groups completed by p0 + T - 1+ const int ng = (int) std::max<int64_t>(0, g_end - g_first);+ if (l.kv_src) {+ if (R == 1) {+ Finish fc;+ fc.w = l.c_norm;+ fc.eps = g.eps;+ finish(d.ckr, T, HD, HD, fc, s);+ CK(cudaMemcpyAsync(d.latent, d.ckr, (size_t) T * HD * 4, cudaMemcpyDeviceToDevice, s));+ } else {+ if (spec_win && l.c_snap != nullptr) { // to replay the accepted rows if drafts are rejected+ const size_t ns = (size_t) R * HD * 4, nr = (size_t) T * HD * 4;+ CK(cudaMemcpyAsync(l.c_snap, l.c_kv, ns, cudaMemcpyDeviceToDevice, s));+ CK(cudaMemcpyAsync(l.c_snap + (size_t) R * HD, l.c_score, ns, cudaMemcpyDeviceToDevice, s));+ CK(cudaMemcpyAsync(l.c_rows, d.ckr, nr, cudaMemcpyDeviceToDevice, s));+ CK(cudaMemcpyAsync(l.c_rows + (size_t) kWin * HD, d.cgr, nr, cudaMemcpyDeviceToDevice, s));+ }+ compress_win(d.ckr, d.cgr, l.c_kv, l.c_score, R, (int) p0, T, HD, l.c_norm, g.eps, d.latent, s);++ dump_sub(d, "ckr", l.il, p0, T, d.ckr, (size_t) T * HD * 4);++ dump_sub(d, "cgr", l.il, p0, T, d.cgr, (size_t) T * HD * 4);+ }+ }+ Rows lim;+ int clen_max = 0;+ for (int t = 0; t < T; ++t) {+ lim.n[t] = (int) ((p0 + t + 1) / R);+ lim.k[t] = std::min(g.ix_topk, lim.n[t]);+ clen_max = std::max(clen_max, lim.n[t]);+ }+ if (l.ix_src) {+ Src& ks = srcs[(size_t) l.ik_src];+ if (l.owns_k && ng > 0) {+ if (!mv(d, {job(l.i_wk, d.latent, d.ikr, ng, HD, g.ix_hd)}, err)) return false;+ Finish fi;+ fi.w = l.ik_norm;+ fi.eps = g.eps;+ fi.rd = g.rd;+ fi.pos = (int) (g_first * R);+ fi.pos_step = R;+ fi.freqs = freqs;+ fi.quant = 3;+ fi.pdst = ks.ik[(size_t) l.dev] + (size_t) g_first * ik_rb();+ fi.pdst_stride = (long long) ik_rb();+ finish(d.ikr, ng, g.ix_hd, g.ix_hd, fi, s);+ if (!wp.par && !replicate(ks.ik, ks.h_ik, ks.ev_ik, fi.pdst, (size_t) g_first * ik_rb(),+ (size_t) ng * ik_rb(), l.dev, err))+ return false;+ }+ if (clen_max > 0) {+ if (!mv(d, {job(l.i_wq_b, d.qr, d.iq, T, g.q_lora, (size_t) g.ix_heads * g.ix_hd)}, err)) return false;+ Finish fi;+ fi.rd = g.rd;+ fi.pos = (int) p0;+ fi.vpp = g.ix_heads;+ fi.freqs = freqs;+ fi.quant = 3;+ finish(d.iq, T * g.ix_heads, g.ix_hd, g.ix_hd, fi, s);+ // (iw rows are 64 apart)+ gemv_f16_rows(l.i_wproj, d.x, (size_t) D, g.ix_heads, D, T,+ 1.0f / std::sqrt((float) g.ix_hd) / std::sqrt((float) g.ix_heads), d.iw, 64, s);+ const long long ss = (long long) o.max_context + 64;+ bool restrict_ = false;+ for (int t = 0; t < T; ++t)+ restrict_ = restrict_ || (lim.n[t] + g.cand_blk - 1) / g.cand_blk > g.cand_topk;+ if (l.uses_cand && restrict_) {+ Rows cl; // the candidate positions of each query: its blocks x blk+ int nmax = 0;+ for (int t = 0; t < T; ++t) {+ cl.n[t] = lim.n[t];+ cl.k[t] = wp.cand[l.il][t];+ nmax = std::max(nmax, wp.cand[l.il][t] * g.cand_blk);+ }+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, nmax, d.cand,+ (int) cand_stride(), g.cand_blk, cl, d.scores, ss, s);+ Rows tr;+ for (int t = 0; t < T; ++t) {+ tr.n[t] = wp.cand[l.il][t] * g.cand_blk;+ tr.k[t] = lim.k[t];+ }+ topk(d.scores, ss, T, tr, d.cand, (int) cand_stride(), g.cand_blk, d.topk, (int) topk_stride(), s);+ } else {+ index_score(d.iq, d.iw, ks.ik[(size_t) l.dev], g.ix_heads, g.ix_hd, T, clen_max, nullptr, 0, 1,+ lim, d.scores, ss, s);+ if (l.cand_src) {+ Rows cr;+ for (int t = 0; t < T; ++t) {+ cr.n[t] = lim.n[t];+ cr.k[t] = g.cand_topk;+ }+ cand_blocks(d.scores, ss, T, cr, g.cand_blk, d.bm, (long long) bm_stride(), d.cand,+ (int) cand_stride(), s);+ }+ topk(d.scores, ss, T, lim, nullptr, 0, 1, d.topk, (int) topk_stride(), s);+ }+ }+ }+ if (l.kv_src && ng > 0) {+ Finish fc;+ fc.rd = g.rd;+ fc.pos = (int) (g_first * R);+ fc.pos_step = R;+ fc.freqs = freqs;+ fc.quant = 2;+ fc.pdst = sr.ckv[(size_t) l.dev] + (size_t) g_first * ckv_rb();+ fc.pdst_stride = (long long) ckv_rb();+ finish(d.latent, ng, HD, HD, fc, s);+ if (!wp.par && !replicate(sr.ckv, sr.h_ckv, sr.ev_ckv, fc.pdst, (size_t) g_first * ckv_rb(),+ (size_t) ng * ckv_rb(), l.dev, err))+ return false;+ if (wp.par && mi == 0 && sr.remote) { // the new rows for the other groups' readers+ Xfer x;+ x.src[0] = fc.pdst;+ x.dst[0] = sr.x_stage;+ x.bytes[0] = (size_t) ng * ckv_rb();+ x.n = 1;+ if (l.owns_k && srcs[(size_t) l.ik_src].ik[(size_t) l.dev] != nullptr) {+ x.src[1] = srcs[(size_t) l.ik_src].ik[(size_t) l.dev] + (size_t) g_first * ik_rb();+ x.dst[1] = sr.x_stage + x_cap() * ckv_rb();+ x.bytes[1] = (size_t) ng * ik_rb();+ x.n = 2;+ }+ xfer_put(x, sr.x_flag, wp.win, d.xcnt, s);+ }+ }++ if (ng > 0) dump_sub(d, "latent", l.il, p0, ng, d.latent, (size_t) ng * HD * 4);++ dump_sub(d, "topk", l.il, p0, T, d.topk, (size_t) T * topk_stride() * 4);+ for (int t = 0; t < T; ++t) ar.nsel[t] = lim.n[t] > 0 ? wp.topk[l.il][t] : 0;+ ckv = sr.ckv[(size_t) l.dev];+ }+ if (l.mtp) { // the block's rows (BF16), after the ring's in ext, as every query's selection+ ckv = d.ext + (size_t) g.win * HD;+ ckv_fmt = kKvBf16;+ iota_rows(d.topk, T, T, (int) topk_stride(), s);+ }+ if (NHL > 0) {+ sparse_attn(d.q, d.ext, ckv, d.topk, (int) topk_stride(), ar, l.sink, NHL, HD, T,+ 1.0f / std::sqrt((float) HD), d.o, s, ckv_fmt, 0, &d.attn_scr);++ dump_sub(d, "o", l.il, p0, T, d.o, (size_t) T * HQ * 4);+ Finish fo;+ fo.rd = g.rd;+ fo.pos = (int) p0;+ fo.vpp = NHL;+ fo.freqs = freqs;+ fo.inverse = 1;+ fo.quant = 1;+ finish(d.o, T * NHL, HD, HD, fo, s);++ dump_sub(d, "o_fin", l.il, p0, T, d.o, (size_t) T * HQ * 4);+ x3::MvJob j[16];+ const int per = HQ / OG;+ for (int gi = 0; gi < OG; ++gi)+ j[gi] = job(l.wo_a[gi], d.o + (size_t) gi * per, d.oa + (size_t) gi * g.o_lora, T, HQ, (size_t) OG * g.o_lora);+ if (!mv_n(d, j, OG, err)) return false;+ if (!mv(d, {job(l.wo_b, d.oa, d.ao, T, (size_t) OG * g.o_lora, D)}, err)) return false;+ } else CK(cudaMemsetAsync(d.ao, 0, (size_t) T * D * sizeof(*d.ao), s)); // (no heads: nothing to add)++ dump_sub(d, "ao", l.il, p0, T, d.ao, (size_t) T * D * 4);+ if (nm > 1) allreduce(d.ao, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);+ hc_post(d.ao, d.h, d.a_post, d.a_comb, D, H, T, d.h2, s);+ std::swap(d.h, d.h2);+ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, 1000 + l.il, p0, T); // (DS4_DUMP_L: after the attention)++ dump_sub(d, "h_attn", l.il, p0, T, d.h, (size_t) T * H * D * 4);+ prof_mark(d, 1);+ // the MoE++#if defined(STRATA_METAL)++ hc_mix_split(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, l.hcf_scale, l.hcf_base, H,++ g.sinkhorn, g.hc_eps, d.f_pre, d.f_post, d.f_comb, s);++#else+ hc_mix_split(d.h, l.hcf_fn, H * D, (2 + H) * H, T, g.eps, d.mix, d.hccnt, l.hcf_scale, l.hcf_base, H, g.sinkhorn, g.hc_eps,+ d.f_pre, d.f_post, d.f_comb, s);++#endif+ hc_pre_norm(d.h, d.a_pre, l.ffn_norm, D, H, T, g.eps, d.x, s);++ dump_sub(d, "f_pre", l.il, p0, T, d.f_pre, (size_t) T * 8 * 4);++ dump_sub(d, "f_post", l.il, p0, T, d.f_post, (size_t) T * 8 * 4);++ dump_sub(d, "f_comb", l.il, p0, T, d.f_comb, (size_t) T * 64 * 4);++ dump_sub(d, "ffn_x", l.il, p0, T, d.x, (size_t) T * D * 4);+ if (wp.async) {+ // the routing goes to the service thread through the mailbox (a group's lead publishes it, the members+ // make the same plan) while the device runs the shared expert and the VRAM experts (from the plan the+ // router made), and waits for the answer only before the experts' sum+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);+ const unsigned sq = wp.seq[l.il];+ const int slot = (int) (sq % kRing);+ const bool lead = mi == 0;+ int na[2] = {0, 0}; // lookahead: the next layers' guesses (their gate on this FFN input), published below+ for (int a = 1; lead && a <= std::min(2, ahead_mode); ++a) {+ const Layer* nx = ahead_layer(L0, a);+ if (nx == nullptr) break;+ const int m = a == 1 ? std::max(1, std::min(kAhead1, nx->k_exp + ahead_extra)) : std::min(kPicks, nx->k_exp);+ float* gl = d.glog_a + (size_t) (a - 1) * kWin * 1024;+ gemv_f32(nx->gate_w, d.x, nx->n_exp, D, T, gl, 1024, s); // (normed with the next layer's FFN+ // weights instead: the same hit rate)+ route_ahead(gl, nx->gate_bias, nx->n_exp, m, T, reqs + slot, (a - 1) * kAhead1, s);+ na[a - 1] = m;+ }+ route_publish(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, l.vram_tab, d.x, D, d.ridx, d.rw,+ d.plan_ptr, lead ? reqs + slot : nullptr, sq, s, l.gate_bias_vl, l.mtp ? 0u : wp.vl, na[0],+ na[1]);+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, FF), job(l.s3, d.x, d.su, T, D, FF)}, err)) return false;+ swiglu(d.sg, d.su, T * FF, g.lim, d.sh, s);+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, FF, D)}, err)) return false;+ prof_mark(d, 2);+ const x3::ExpertLayout& lay = nm > 1 ? l.hlay : l.lay;+ if (T == 1 && moe_token_path) {+ x3::MoeArgs ma;+ ma.plan_ptr = d.plan_ptr;+ ma.plan_w = d.rw;+ ma.cpu_part = lead ? resps[slot].part[0] : nullptr;+ ma.cpu_flag = lead ? &resps[slot].cpu[0] : nullptr;+ ma.lay = lay;+ ma.k = l.k_exp;+ ma.n_embd = D;+ ma.n_ff = FF;+ ma.limit = g.lim;+ const bool mx = lay.fmt == gg::kFmtMXFP4;+ if (zc_stage_buf != nullptr && l.zc_tab != nullptr) // (the store's experts copied to VRAM at the PCIe rate first)+ zc_stage(d.plan_ptr, d.ridx, l.zc_tab, l.k_exp, zc_stage_buf, zc_stride, l.blob, s);++ // DS4_TLOG2: the expert stage's three host spans (gate/up issue, the CPU-lane wait, down issue)++ static const bool tlog2 = std::getenv("DS4_TLOG2") != nullptr;++ static double t_moe_gu = 0, t_moe_wait = 0, t_moe_dn = 0; static long t_moe_n = 0; static bool t_moe_reg = false;++ if (tlog2 && !t_moe_reg) {++ t_moe_reg = true;++ std::atexit([] {++ std::fprintf(stderr, "tlog2: %ld stages | host ms a stage: gate_up %.2f cpu_wait %.2f down %.2f\n",++ t_moe_n, t_moe_n ? t_moe_gu / (double) t_moe_n : 0.0,++ t_moe_n ? t_moe_wait / (double) t_moe_n : 0.0,++ t_moe_n ? t_moe_dn / (double) t_moe_n : 0.0);++ });++ }++ const double m0 = tlog2 ? now_ms() : 0.0;+ if (mx) gg::moe_gate_up(ma, d.x, d.xg, s);+ else x3::moe_gate_up(ma, d.x, d.xg, s);++ const double m1 = tlog2 ? now_ms() : 0.0;+ prof_mark(d, 3);+ if (lead) wait_seq(&resps[slot].seq, sq, s);++ const double m2 = tlog2 ? now_ms() : 0.0;+ prof_mark(d, 4);+ if (mx) gg::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);+ else x3::moe_down(ma, d.xg, d.sh_out, d.moe_out, s);++ if (tlog2) { t_moe_gu += m1 - m0; t_moe_wait += m2 - m1; t_moe_dn += now_ms() - m2; ++t_moe_n; }+ } else { // each distinct expert read once for the window's tokens+ gf::MoeWin mw;+ mw.plan_ptr = d.plan_ptr;+ mw.T = T;+ mw.k = l.k_exp;+ mw.grp_ptr = d.grp_ptr;+ mw.grp_start = d.grp_start;+ mw.grp_ent = d.grp_ent;+ mw.n_groups = d.n_groups;+ gf::moe_win_group(mw, s);+ x3::WinArgs wa;+ wa.grp_ptr = d.grp_ptr;+ wa.grp_start = d.grp_start;+ wa.grp_ent = d.grp_ent;+ wa.n_groups = d.n_groups;+ wa.plan_ptr = d.plan_ptr;+ wa.plan_w = d.rw;+ wa.cpu_part = lead ? &resps[slot].part[0][0] : nullptr;+ wa.cpu_stride = kMaxDim;+ wa.cpu_flag = lead ? resps[slot].cpu : nullptr;+ wa.wait_ptr = lead ? &resps[slot].seq : nullptr;+ wa.wait_val = sq;+ wa.x = d.x;+ wa.x_stride = (size_t) D;+ wa.h = d.xg;+ wa.part = d.ed;+ wa.sh_out = d.sh_out;+ wa.out = d.moe_out;+ wa.out_stride = (size_t) D;+ wa.lay = lay;+ wa.T = T;+ wa.k = l.k_exp;+ wa.n_embd = D;+ wa.n_ff = FF;+ wa.limit = g.lim;+ if (!(lay.fmt == gg::kFmtMXFP4 ? gg::win_experts(wa, s) : x3::win_experts(wa, s))) {+ err = "exl3 win_experts failed";+ return false;+ }+ }+ if (lead) st.experts += (int64_t) T * l.k_exp;+ if (nm > 1) allreduce(d.moe_out, T * D, ag, mi, ++d.ar_seq, d.xcnt, s);+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);+ std::swap(d.h, d.h2);+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));+ prof_mark(d, 5);+ return true;+ }+ if (nm > 1) {+ err = "a tensor-parallel layer on the host-routed path";+ return false;+ }+ // (glog rows are 1024 apart: route reads them so)++ static const bool phases = std::getenv("DS4_PHASES") != nullptr; // debugging: where a decode layer's host time goes++ static double ph[8] = {0}, ph_prev = 0; static long ph_n = 0;++ const double ph0 = phases ? now_ms() : 0.0;+ gemv_f32(l.gate_w, d.x, l.n_exp, D, T, d.glog, 1024, s);+ VlBits vb = {};+ vb.w[0] = l.mtp ? 0u : wp.vl;+ route(d.glog, l.gate_bias, l.n_exp, l.k_exp, T, g.route_scale, d.ridx, d.rw, s, l.gate_bias_vl,+ vb.w[0] != 0 ? &vb : nullptr);+ CK(cudaMemcpyAsync(h_idx, d.ridx, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaMemcpyAsync(h_w, d.rw, (size_t) T * kPicks * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaMemcpyAsync(h_x, d.x, (size_t) T * D * 4, cudaMemcpyDeviceToHost, s));+ CK(cudaEventRecord(d.ev, s));+ // the shared expert while the host looks at the picks+ if (!mv(d, {job(l.s1, d.x, d.sg, T, D, g.ff), job(l.s3, d.x, d.su, T, D, g.ff)}, err)) return false;++#if defined(STRATA_METAL)++ int la_n = 0; // the next layers' guessed picks (their gate on this FFN input), on the GPU: a gemv and a route a layer++ if (T == 1 && la_enabled()) {++ for (int a = 1; a <= la_depth(); ++a) {++ const int il = l.il + a;++ if (il >= g.nl) break;++ const Layer& nx = L[(size_t) il];++ if (nx.nm != 1 || nx.lay.fmt != gg::kFmtMXFP4 || nx.gate_bias == nullptr || nx.gate_w == nullptr) break;++ gemv_f32(nx.gate_w, d.x, nx.n_exp, D, 1, d.glog_a + (size_t) (a - 1) * kWin * 1024, 1024, s);++ route(d.glog_a + (size_t) (a - 1) * kWin * 1024, nx.gate_bias, nx.n_exp, nx.k_exp, 1, g.route_scale, d.ridx_a + (a - 1) * kPicks,++ d.rw_a + (a - 1) * kPicks, s, nullptr, nullptr);++ CK(cudaMemcpyAsync(h_idx_a + (a - 1) * kPicks, d.ridx_a + (a - 1) * kPicks, (size_t) kPicks * 4, cudaMemcpyDeviceToHost, s));++ la_n = a;++ }++ if (la_n > 0) CK(cudaEventRecord(d.ev2, s));++ }++#endif++ if (std::getenv("DS4_DBG_MOE")) { dbg_nf("x", l.il, d.x, (size_t) T * D); dbg_nf("sg", l.il, d.sg, (size_t) T * g.ff); dbg_nf("su", l.il, d.su, (size_t) T * g.ff);++ std::fprintf(stderr, "DBG s1 fmt=%d K=%d k=%d n=%d ld=%d raw=%p\n", l.s1.fmt, l.s1.K, l.s1.k, l.s1.n, l.s1.ld, (void*) l.s1.raw); }+ swiglu(d.sg, d.su, T * g.ff, g.lim, d.sh, s);++ if (std::getenv("DS4_DBG_MOE")) dbg_nf("sh", l.il, d.sh, (size_t) T * g.ff);+ if (!mv(d, {job(l.s2, d.sh, d.sh_out, T, g.ff, D)}, err)) return false;+ const double tw = now_ms();+ CK(cudaEventSynchronize(d.ev));+ st.gpu_wait_ms += now_ms() - tw;++ const double ph2 = phases ? now_ms() : 0.0;++ dump_sub(d, "glog", l.il, p0, T, d.glog, (size_t) T * 1024 * 4);++ dump_sub_host("ridx", l.il, p0, T, h_idx, (size_t) T * kPicks * 4);++ dump_sub_host("rw", l.il, p0, T, h_w, (size_t) T * kPicks * 4);++ dump_sub(d, "sh_out", l.il, p0, T, d.sh_out, (size_t) T * D * 4);+ // the picks by expert: the VRAM ones' rows (gathered inputs, one GEMV per matrix for all their tokens), the+ // CPU lane's entries+ Combine cm;+ Gather ga;+ uint32_t* pr = prof.data() + (size_t) l.il * g.n_exp;+ int vex[kWin * kPicks], vrow0[kWin * kPicks], vn[kWin * kPicks];+ int nvex = 0, nrows = 0;+ std::vector<Ent> ents;+ std::vector<std::pair<int, float>> cpu_rows; // slot -> (token, weight)+ std::vector<std::vector<std::pair<int, int>>> per((size_t) l.n_exp); // expert -> (token, pick)+ for (int t = 0; t < T; ++t) {+ cm.n[t] = 0;+ cm.cpu[t] = 0;+ for (int i = 0; i < l.k_exp; ++i) {+ const int e = h_idx[t * kPicks + i];+ if (e < 0 || e >= l.n_exp) {- err = "router picked expert " + std::to_string(e);++ err = "router picked expert " + std::to_string(e) + " (layer " + std::to_string(l.il) + ", token " + std::to_string(t) + ", pick " + std::to_string(i) + ", of " + std::to_string(l.k_exp) + ")";+ return false;+ }+ ++pr[e];+ l.heat[(size_t) e] += 1.0f;+ if (l.slot[(size_t) e] < 0) l.cpu_hits.push_back(e);+ per[(size_t) e].push_back({t, i});+ }+ }+ int nslots = 0;+ for (int e = 0; e < l.n_exp; ++e) {+ const auto& pe = per[(size_t) e];+ if (pe.empty()) continue;- if (l.slot[(size_t) e] >= 0) {++#if defined(STRATA_METAL)++ const bool on_gpu = l.slot[(size_t) e] >= 0 || (T == 1 && zc_try(l, e));++#else++ const bool on_gpu = l.slot[(size_t) e] >= 0;++#endif++ if (on_gpu) {+ vex[nvex] = e;+ vrow0[nvex] = nrows;+ vn[nvex] = (int) pe.size();+ for (const auto& tp : pe) {+ ga.src[nrows] = tp.first;+ const int t = tp.first;+ cm.row[t][cm.n[t]] = nrows;+ cm.w[t][cm.n[t]] = h_w[t * kPicks + tp.second];+ ++cm.n[t];+ ++nrows;+ }+ ++nvex;+ } else {+ for (size_t a = 0; a < pe.size(); a += (size_t) ent_cap(l)) {+ Ent en;+ en.e = e;+ en.slot0 = nslots;+ en.n = (int) std::min<size_t>((size_t) ent_cap(l), pe.size() - a);+ for (int i = 0; i < en.n; ++i) {+ en.tok[i] = pe[a + (size_t) i].first;+ cpu_rows.push_back({pe[a + (size_t) i].first, h_w[pe[a + (size_t) i].first * kPicks + pe[a + (size_t) i].second]});+ cm.cpu[en.tok[i]] = 1;+ }+ nslots += en.n;+ ents.push_back(en);+ }+ }+ }+ st.experts += (int64_t) T * l.k_exp;+ st.cpu_experts += nslots;++#if defined(STRATA_METAL)++ if (T == 1) lookahead_prefetch(l, d.ev2, la_n, ents);++#endif+ if (nrows > 0) {+ gather_rows(d.x, ga, nrows, D, d.xe, s);++ dump_sub(d, "xe", l.il, p0, nrows, d.xe, (size_t) nrows * D * 4);++ { std::vector<int> vmap; for (int i = 0; i < nvex; ++i) { vmap.push_back(vex[i]); vmap.push_back(vrow0[i]); vmap.push_back(vn[i]); }++ dump_sub_host("vmap", l.il, p0, nvex, vmap.data(), vmap.size() * 4); }+ x3::MvJob j[2 * kWin * kPicks];+ int nj = 0;+ for (int i = 0; i < nvex; ++i) {+ const auto& vm = l.vm[(size_t) vex[i]];+ j[nj++] = job(vm[0], d.xe + (size_t) vrow0[i] * D, d.xg + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);+ j[nj++] = job(vm[1], d.xe + (size_t) vrow0[i] * D, d.xu + (size_t) vrow0[i] * g.ff, vn[i], D, g.ff);+ }+ if (!mv_n(d, j, nj, err)) return false;++ dump_sub(d, "xg", l.il, p0, nrows, d.xg, (size_t) nrows * g.ff * 4);++ dump_sub(d, "xu", l.il, p0, nrows, d.xu, (size_t) nrows * g.ff * 4);+ swiglu(d.xg, d.xu, nrows * g.ff, g.lim, d.xhh, s);++ dump_sub(d, "xhh", l.il, p0, nrows, d.xhh, (size_t) nrows * g.ff * 4);+ nj = 0;+ for (int i = 0; i < nvex; ++i)+ j[nj++] = job(l.vm[(size_t) vex[i]][2], d.xhh + (size_t) vrow0[i] * g.ff, d.ed + (size_t) vrow0[i] * D,+ vn[i], g.ff, D);+ if (!mv_n(d, j, nj, err)) return false;++ dump_sub(d, "ed", l.il, p0, nrows, d.ed, (size_t) nrows * D * 4);+ }++ const double ph3 = phases ? now_ms() : 0.0;+ if (!ents.empty()) {+ const double tc = now_ms();- cpu_experts(l, h_x, ents);++#if defined(STRATA_METAL)++ ds4_metal_kick(); // (the VRAM experts run on the GPU while the CPU lane works)++ ds4_busy_begin(); // (and the GPU is kept out of its idle clocks until the lane is done)++#endif++ static const bool skip_cpu = std::getenv("DS4_SKIP_CPU") != nullptr; // (timing only: the CPU lane's experts left out)++ if (!skip_cpu) cpu_experts(l, h_x, ents);++ else if (const char* fk = std::getenv("DS4_FAKE_CPU_US")) std::this_thread::sleep_for(std::chrono::microseconds(std::atoi(fk))); // (timing only)++#if defined(STRATA_METAL)++ ds4_busy_end();++#endif+ std::fill(h_part, h_part + (size_t) T * D, 0.0f);+ for (int sl = 0; sl < nslots; ++sl) {+ float* o = h_part + (size_t) cpu_rows[(size_t) sl].first * D;+ const float wv = cpu_rows[(size_t) sl].second;+ const float* y = c_yd.data() + (size_t) sl * D;+ for (int e = 0; e < D; ++e) o[e] += wv * y[e];+ }++ { static double ms = 0; static long calls = 0, exps = 0; static bool reg = false;++ ms += now_ms() - tc; ++calls; exps += (long) ents.size();++ if (std::getenv("DS4_CPU_LOG") && !reg) { reg = true; std::atexit([] { std::fprintf(stderr, "cpu lane calls: %ld, experts %ld, %.1f ms (%.3f ms a call, %.3f ms an expert)%c", calls, exps, ms, ms / (double) calls, ms / (double) exps, 10); }); }++ (void) 0; (void) ms; }+ st.cpu_ms += now_ms() - tc;++ dump_sub_host("cpu_part", l.il, p0, T, h_part, (size_t) T * D * 4);+ CK(cudaMemcpyAsync(d.cpu_part, h_part, (size_t) T * D * 4, cudaMemcpyHostToDevice, s));+ }+ combine(d.ed, ents.empty() ? nullptr : d.cpu_part, d.sh_out, cm, T, D, d.moe_out, s);++ dump_sub(d, "moe_out", l.il, p0, T, d.moe_out, (size_t) T * D * 4);++ if (std::getenv("DS4_DBG_MOE")) {++ float hp = 0.0f;++ for (size_t i = 0; i < (size_t) T * D; ++i) hp = std::max(hp, std::fabs(h_part[i]));++ std::fprintf(stderr, "DBG moe il=%d T=%d vram_rows=%d cpu_slots=%d max|h_part|=%.4g", l.il, T, nrows, nslots, (double) hp);++ dbg_nf("moe_out", l.il, d.moe_out, (size_t) T * D);++ dbg_nf("sh_out", l.il, d.sh_out, (size_t) T * D);++ }+ hc_post(d.moe_out, d.h, d.f_post, d.f_comb, D, H, T, d.h2, s);+ std::swap(d.h, d.h2);++ dump_sub(d, "h_out", l.il, p0, T, d.h, (size_t) T * H * D * 4);++ if (dump_l != nullptr && mi == 0) dump_layer(d, d.h, l.il, p0, T); // (post-layer residual, as the chunk path writes it)+ CK(cudaMemcpyAsync(d.pre, d.f_pre, (size_t) T * 8 * 4, cudaMemcpyDeviceToDevice, s));++ if (phases && T == 1) {++ const double ph4 = now_ms();++ ph[0] += ph2 - ph0; ph[1] += ph3 - ph2; ph[2] += ph4 - ph3; if (ph_prev > 0 && ph0 - ph_prev < 5.0) ph[3] += ph0 - ph_prev; ph_prev = ph4; ++ph_n;++ static bool reg = false;++ if (!reg) { reg = true; std::atexit([] { std::fprintf(stderr, "phases per layer call (%ld): route enqueue + shared expert + wait %.3f ms, routing analysis + GPU experts enqueue %.3f ms, CPU lane + combine + hc_post %.3f ms, attention enqueue %.3f ms%s", ph_n, ph[0] / ph_n, ph[1] / ph_n, ph[2] / ph_n, ph[3] / ph_n, "\n"); }); }++ }+ // (h_part / h_x are written again only after the next layer's route has synchronized, which follows these)+ return true;+ }++ int sample(float temperature, float top_p, int top_k, float min_p, uint64_t seed, uint64_t counter) {+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ strata::dsv41k::SamplerParams sp;+ sp.temperature = temperature;+ sp.greedy = temperature <= 0.0f;+ sp.top_p = top_p;+ sp.top_k = top_k;+ sp.min_p = min_p;+ sp.seed = seed;+ sp.counter = counter;+ strata::dsv41k::sample_tokens(d.logits + (size_t) std::max(0, n_logits - 1) * g.vocab, 1, g.vocab, nullptr, 0,+ sp, d.tok, d.st);+ int t = -1;++#if defined(STRATA_METAL)++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)++#endif+ cudaMemcpyAsync(&t, d.tok, 4, cudaMemcpyDeviceToHost, d.st);+ cudaStreamSynchronize(d.st);+ return t;+ }++ bool logits(std::vector<float>& out) {+ if (logits_all) {+ out = chunk_logits;+ return true;+ }+ Dev& d = devs[(size_t) head_dev];+ cudaSetDevice(d.id);+ out.resize((size_t) std::max(1, n_logits) * g.vocab);++#if defined(STRATA_METAL)++ ds4_metal_flush(); // (a host read of a device result: commit the batched dispatches first)++#endif+ return cudaMemcpy(out.data(), d.logits, out.size() * 4, cudaMemcpyDeviceToHost) == cudaSuccess;+ }++ void save_profile() const {- if (o.profile.empty()) return;++ if (o.profile.empty() || o.profile_readonly) return; // (--profile-readonly: a test keeps the file it read)+ const std::string tmp = o.profile + ".tmp";+ std::FILE* f = std::fopen(tmp.c_str(), "wb");+ if (f == nullptr) return;+ const bool ok = std::fwrite(prof.data(), 4, prof.size(), f) == prof.size();+ std::fclose(f);+ // (the old profile replaced: std::rename does not replace an existing file on Windows, so there the profile+ // never moved on and only .tmp files were left)+ if (!ok || !os::replace_file(tmp, o.profile)) std::remove(tmp.c_str());+ }++ std::vector<std::string> info_lines() const {+ std::vector<std::string> v;+ int nh = 0;+ for (const Layer& l : L) nh += l.n_exp - l.n_vram;+ char b[512];+ // kv=fp4: DeepSeek's own cache layout (the compressed KV and the index keys FP4, the window FP8)+ std::snprintf(b, sizeof b, "INFO context=%lld kv=fp4 kv_resident=%lld expert_slots=%d engine_kind=ds4 "+ "cpu_experts=%d ram_experts=%d engine=ds4-0.1",+ (long long) o.max_context, (long long) o.max_context, g.nl * g.n_exp - nh, nh, ram_held);+ v.push_back(b);+ if (commit_kept > 0) { // (Windows, an integrated GPU: the commit kept experts off the GPU - the server says so)+ std::snprintf(b, sizeof b,+ "NOTE experts: %.0f GB stay off the GPU - Windows' page file is too small to hold them there; "+ "a page file of %.0f GB keeps them on the GPU (System > About > Advanced system settings > "+ "Performance > Advanced > Virtual memory), then restart Maya", commit_kept / 1e9, pagefile_hint);+ v.push_back(b);+ std::snprintf(b, sizeof b, "INFO gpu_commit_capped_gb=%.0f pagefile_hint_gb=%.0f", std::ceil(commit_kept / 1e9),+ pagefile_hint);+ v.push_back(b);+ }+ if (pk.eg_layers > 0) { // the n-gram memory's (Engram's) table rows a token reads, for the Monitor+ std::snprintf(b, sizeof b, "INFO engram_layers=%d engram_lookups=%d", pk.eg_layers,+ pk.eg_layers * (pk.eg_ngram - 1) * pk.eg_heads);+ v.push_back(b);+ }+ if (img_tok >= 0) { // the server's image resize (DeepSeek's image_processor.py)+ std::snprintf(b, sizeof b,+ "INFO vision=ds4 vision_patch=%d vision_merge=%d vision_min_pixels=%lld vision_max_tokens=%d "+ "vision_max_wh_ratio=%lld image_token=%d vision_gpu=%d vision_mib=%lld",+ vg.patch, vg.merge, (long long) pk.num("vision.min_pixels", 0), vg.max_tokens,+ (long long) pk.num("vision.max_wh_ratio", 0), img_tok, vt.device(),+ (long long) (vt.device_bytes() >> 20));+ v.push_back(b);+ } else if (!vision_err.empty()) { // (the server offers no pictures and says why)+ std::string why = vision_err;+ for (char& ch : why)+ if (ch == '\n' || ch == '\r') ch = ' ';+ v.push_back("NOTE images: OFF - the model's vision tower did not load (" + why + ")");+ v.push_back("INFO vision=off");+ }+ return v;+ }+};++Model::Model() : p_(new Impl) {}+Model::~Model() = default;+bool Model::load(const Options& o, std::string& err) { return p_->load(o, err); }+bool Model::step(int token, bool logits, std::string& err) { return p_->step(token, logits, err); }+bool Model::feed(const int* tokens, int n, int logits_rows, std::string& err) {+ Impl& p = *p_;+ p.pm.last = -1;+ p.halted = false;++ if (std::getenv("DS4_DBG_CHUNK")) std::fprintf(stderr, "feed n %d chunk %d chunk_min %d\n", n, p.chunk, p.chunk_min);+ if (p.chunk > 0 && n >= p.chunk_min) { // a long run - or one the flow could take, if it is the faster path here+ const double t0 = now_ms(); // (every row's logits (tests): the chunked path, which keeps them all)+ const bool ok = logits_rows > kWin || p.pick_path(n) ? p.forward_chunks(tokens, n, logits_rows, err)+ : p.feed_windows(tokens, n, logits_rows, err);+ if (ok) p.path_done(now_ms() - t0);+ return ok;+ }+ return p.feed_windows(tokens, n, logits_rows, err);+}+bool Model::feed_mark(const int* tokens, int n, int logits_rows, int slot, std::string& err) {+ Impl& p = *p_;+ p.pm.last = -1;+ p.halted = false;+ const double t0 = now_ms();+ if (n >= 2 && p.chunk > 0 && n >= p.chunk_min && p.pick_path(n)) { // (the last token rides along; its layers+ const bool ok = p.forward_chunks(tokens, n, logits_rows, err, slot); // keep the state before it)+ if (ok) p.path_done(now_ms() - t0);+ return ok;+ }+ if (n > 1 && !p.feed_windows(tokens, n - 1, 0, err)) return false;+ p.checkpoint(slot);+ const bool ok = n < 1 || p.feed_windows(tokens + n - 1, 1, logits_rows, err);+ if (ok) p.path_done(now_ms() - t0); // (windows, when pick_path chose them)+ return ok;+}+int Model::feed_unit() const { return p_->chunk > 0 ? std::max(p_->chunk, (int) p_->o.max_context) : kWin; }+void Model::set_progress(std::function<void(int64_t)> f) { p_->progress = std::move(f); }+void Model::set_cancel(std::function<bool()> f) { p_->cancel = std::move(f); }+bool Model::cancelled() const { return p_->halted; }+int Model::sample(float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter) {+ return p_->sample(t, tp, tk, mp, seed, counter);+}+bool Model::logits(std::vector<float>& out) { return p_->logits(out); }+int Model::image_token() const { return p_->img_tok; }+int Model::dim() const { return p_->g.dim; }+bool Model::encode_image(const uint8_t* rgb, int width, int height, std::vector<uint16_t>& span, std::string& err) {+ return p_->encode_image(rgb, width, height, span, err);+}+void Model::set_images(std::vector<Image> images) {+ std::sort(images.begin(), images.end(), [](const Image& a, const Image& b) { return a.start < b.start; });+ p_->imgs = std::move(images);+}+int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }+bool Model::gpus_are_gfx12() const { return p_->rdna4(); }+bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,+ float mp, uint64_t seed, uint64_t counter) {+ return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);+}+bool Model::verify(int y, const std::vector<int>& d, float t, float tp, int tk, float mp, uint64_t seed, uint64_t counter,+ int& accepted, int& next, std::string& err) {+ return p_->verify(y, d, t, tp, tk, mp, seed, counter, accepted, next, err);+}+int64_t Model::pos() const { return p_->pos; }+int64_t Model::max_context() const { return p_->o.max_context; }+int Model::vocab() const { return p_->g.vocab; }+const std::vector<int32_t>& Model::tokens() const { return p_->toks; }+void Model::reset() { p_->reset(); }+void Model::checkpoint(int slot) { p_->checkpoint(slot); }+bool Model::restore(int64_t limit) { return p_->restore(limit); }+void Model::pin() { p_->pin(); }+std::vector<int64_t> Model::check_positions() const { return p_->check_positions(); }+int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }+uint64_t Model::slot_save(std::vector<uint8_t>& out, std::string& err) { return p_->slot_save(out, err); }+bool Model::slot_load(const uint8_t* data, size_t bytes, int64_t n_pos, std::string& err) {+ return p_->slot_load(data, bytes, n_pos, err);+}+std::string Model::slot_identity() const { return p_->slot_identity(); }+void Model::save_profile() const { p_->save_profile(); }+const Stats& Model::stats() const {+ Stats& st = p_->st;+ if (p_->store) { // (the store keeps its own counters: copied in when asked)+ const ExpertStore& x = *p_->store;+ st.store_hits = x.stats().hits.load();+ st.store_misses = x.stats().misses.load();+ st.store_prefetched = x.stats().prefetched.load();+ st.store_pf_used = x.stats().prefetch_used.load();+ st.store_pf_wasted = x.stats().prefetch_wasted.load();+ st.store_evictions = x.stats().evictions.load();+ st.store_bytes = x.stats().bytes_read.load();+ st.store_fails = x.stats().fails.load();+ st.store_wait_ms = x.stats().wait_ns.load() / 1e6;+ st.store_busy_ms = x.busy_ms();+ st.store_zc_visible = (int64_t) x.visible_count();+ st.store_slots = (int64_t) x.slots_allocated();+ st.store_slots_max = (int64_t) x.slots_max();+ }+ return st;+}+std::string Model::describe() const { return p_->describe(); }+std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }++} // namespace strata::ds4diff --cc src/dsv41/ds4_model.hppindex afa9d2a69,f8b087c70..6cf65dbad--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@@ -30,6 -29,6 +31,9 @@@ struct Optionsdouble vram_experts = 1.0; // the share of the expert budget to use (tests: less, so the CPU lane works)int cpu_threads = 0; // 0: the physical coresstd::string profile; // expert hit counts (read to place the hottest experts in VRAM, written back)++ // --profile-readonly: read the profile to place the experts, never write it back. Without this the run's routing++ // counts go into the file, so the next run places experts differently and greedy output / KL move between runs.++ bool profile_readonly = false;bool expert_cache = true; // experts move between VRAM and the CPU lane as they are used// the CPU lane's experts read in place from the model's files, the page cache keeping what fits, instead of// copied into RAM: 1 on, 0 off, -1 (auto) where the RAM this process can get does not hold themdiff --cc src/dsv41/ds4_tq.cuindex 724072e67,10c7261f8..a55759404--- a/src/dsv41/ds4_tq.cu+++ b/src/dsv41/ds4_tq.cu@@@ -10,6 -10,6 +10,7 @@@// block scale d is the lane's own. Lane l = r + 16 kg owns row r and the 16 steps 16 kg .. 16 kg + 15 of every block, whose// 15-bit states are one 128-bit window of the stream (the tile code of tools/dsv41_lab/wmma_emu.hpp, checked there against a// double reference and on the card by tests/dsv41/tqw_hip_test.cpp). Elsewhere the same lane mapping in plain FMAs.++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "ds4_iq.cuh"#include <cuda_fp16.h>diff --cc src/dsv41/ds4_vision.cuindex 38916c755,1d94ebe4b..ec112ca5c--- a/src/dsv41/ds4_vision.cu+++ b/src/dsv41/ds4_vision.cu@@@ -10,6 -10,6 +10,7 @@@// x[piece] += wo(O) + bias// xn = norm2(x); per row piece: x += w2(silu(gate) * up), gate | up = w1(xn)// xn = norm(x); the 3 x 3 unfold; GELU(w1 + b1); w2 + b2 -> BF16 rows (the aligner's weights stream in pieces too)++#include "ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "ds4_vision.cuh"#include <cublas_v2.h>diff --cc src/dsv41/vendor/exl3.cuindex 501010668,01bf0cd44..e94ffd7b6--- a/src/dsv41/vendor/exl3.cu+++ b/src/dsv41/vendor/exl3.cu@@@ -19,6 -19,9 +19,10 @@@// NVIDIA GPUs before sm_80 (V100, T4, RTX 20) decode with v1.1.8's kernels instead (exl3_legacy.cu, use_legacy// below): this file's CUDA-core fallback was slower there (a V100 with EXL3-3.05bpw: decode 4.9 -> 4.4 tok/s). The// prompt path below is this file's on every GPU.+ #if defined(STRATA_METAL)++#include "../ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal+ #include <ds4_metal_host.h>+ #endif#include "strata/dsv41/exl3.hpp"#include <cuda_fp16.h>@@@ -1266,7 -1269,7 +1270,10 @@@ int sm_count()// token path and the verify windows take the same family on a device, so a window stays bit for bit its token path.// Asked once per device.bool use_legacy() {--#if defined(STRATA_USE_HIP)++#if defined(STRATA_USE_HIP) || defined(STRATA_METAL)++ // HIP has no v1.1.8 kernels. Metal has neither them nor a compute capability to ask for: the shim answers++ // cudaDeviceGetAttribute with an error, so `major` stayed 0 and every Metal run without STRATA_GLM_EXL3_DECODE=new++ // took this branch and aborted in exl3::legacy::prepare's stub. The 2.25 bpw pack never uses EXL3.return false;#elsestatic std::atomic<int> cache[64]; // 0 not asked, 1 v1.1.8's kernels, 2 this file'sdiff --cc src/dsv41/vendor/exl3_legacy.cuindex bcc389b1e,0a028bbda..a35a9a1f4--- a/src/dsv41/vendor/exl3_legacy.cu+++ b/src/dsv41/vendor/exl3_legacy.cu@@@ -19,6 -19,6 +19,7 @@@// blocks: the partial sums go to a per-stream scratch and the LAST block of an output block adds them up in// split order (deterministic) and runs the output rotation.#if !defined(STRATA_USE_HIP) // (AMD runs exl3.cu's kernels; EXL3 does not run there anyway)++#include "../ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "strata/dsv41/exl3.hpp"#include <cuda_fp16.h>diff --cc src/dsv41/vendor/moe_win.cuindex fadbad001,1395738d0..6254e1fd1--- a/src/dsv41/vendor/moe_win.cu+++ b/src/dsv41/vendor/moe_win.cu@@@ -2,6 -2,6 +2,7 @@@// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.// src/dsv41/vendor/moe_win.cu - the window's expert grouping kernel.// Extracted from Project Maya's src/kernels/cuda/glm_fast.cu (MIT, see third_party/project-maya/NOTICE.md).++#include "../ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "strata/dsv41/moe_win.hpp"#include <cstdio>#include <cstdlib>diff --cc src/dsv41/vendor/sampler.cuindex b406df584,302637ec7..60062fbd5--- a/src/dsv41/vendor/sampler.cu+++ b/src/dsv41/vendor/sampler.cu@@@ -15,11 -15,19 +15,20 @@@// Both kernels put ONE BLOCK per token over the vocabulary: `sampler_greedy_kernel` is the plain argmax,// `sampler_kernel` runs the sampled chain as `top_k` block-argmax rounds followed by the top_p / temperature /// draw chain (its header says why the selection must be parallel and why the tie rule keeps the semantics).++#include "../ds4_launch.h" // strata_ds4_launch: <<<>>> on CUDA and HIP, a dispatch on Metal#include "strata/dsv41/sampler.hpp"+ #if defined(STRATA_METAL)+ #include <ds4_metal_host.h>+ #endif#include <cuda_runtime.h>+ #if defined(STRATA_METAL)+ #include <strata/gpu/runtime.hpp>+ #endif+ #include <algorithm>#include <cmath>+ #include <vector>#include <cstdio>#include <cstdlib>diff --cc strata_control/dsv41_fit.pyindex ef7f354d8,f9dcbaeb0..69492c15f--- a/strata_control/dsv41_fit.py+++ b/strata_control/dsv41_fit.py@@@ -478,3 -226,3 +478,87 @@@ def options(cards: list[dict], ram_gb:opts[0]["recommended"] = Trueout["options"], out["recommended"] = opts, "all"return out++++++# ---------------------------------------------------------------------------------------------------------------------++# Apple Silicon: one pool of unified memory, the routed experts are file-backed and stay resident up to the engine's cap.++# The numbers are the three GSQ-RCO packs (the research plan's table, measured from the public allocations) and the++# engine's own cap rule (src/dsv41/ds4_model.cu metal_expert_cap, metal/src/engine/arena_budget.h).++++MAC_PACKS: dict[str, dict[str, Any]] = {++ "IQ1_M": {"name": "1.75 bpw (IQ1_M)", "routed_gb": 114.56, "backbone_gb": 120.72, "bpw": 1.75},++ "IQ2_XXS": {"name": "2.00 bpw (IQ2_XXS), the default for most users", "routed_gb": 132.07, "backbone_gb": 138.24, "bpw": 2.00},++ "IQ2_XS": {"name": "2.25 bpw (IQ2_XS), the quality one", "routed_gb": 149.06, "backbone_gb": 155.23, "bpw": 2.25},++}++MAC_ENGRAM_GB = 110.60 # the same Engram table for all three, read row by row off the SSD++MAC_ENGINE_FOOTPRINT_GB = 10.0 # the process' anonymous memory after the load (Metal buffers, host buffers, the KV cache)++MAC_DEVICE_SHARE = 0.75 # recommendedMaxWorkingSetSize as a share of the RAM (107.5 of 128 GB on the M5 Max)++MAC_FILECACHE_FLOOR_GB = 37.2 # vm.vm_page_filecache_min on the 128 GB M5 Max++MAC_HEADROOM_GB = 12.0 # beside that floor: macOS itself and one more program++MAC_MEASURED = ("M5 Max, 128 GB, 2.25 bpw, 2026-10-25: 14.4-15.8 tokens/s for an answer (12.4-12.6 on agent "++ "work), a 3,000-token prompt at 380-382 tokens/s, 32,000 at 306; the engine kept 71.4 GB of the "++ "experts resident and read 93.4 MB a token from the SSD")++++++def mac_expert_cap_gb(ram_gb: float) -> float:++ """The expert budget the engine will pick on a Mac of `ram_gb` (GiB, as the rest of the app reports memory), in++ decimal GB: the RAM less what macOS keeps, less the process' own footprint. Mirrors metal_expert_cap() with its++ big-Mac rule (>= 64 GB: 40% left, at least the file-cache floor + 12 GB). 128 GiB gives the 71.4 GB measured on++ the M5 Max."""++ gib = 1.073741824++ p = float(ram_gb) * gib++ if p <= 0:++ return 0.0++ keep = max(7.0, 0.15 * p)++ leave = max(keep, 0.40 * p, MAC_FILECACHE_FLOOR_GB + MAC_HEADROOM_GB) if p >= 64.0 else keep++ cap = min(p - leave, MAC_DEVICE_SHARE * p)++ return max(0.0, cap - MAC_ENGINE_FOOTPRINT_GB - 1.0)++++++def mac_plan(ram_gb: float, quant: str = "IQ2_XS", disk_free_gb: float | None = None) -> dict[str, Any]:++ """Where the DeepSeek pack goes on an Apple Silicon Mac. Keys: quant, name, routed_gb, dense_gb, resident_gb,++ streaming_gb, engram_gb, disk_need_gb, disk_ok, ram_gb, verdict (fits / streams / tight / small), lines, warnings,++ words, measured. `fits` means every routed expert stays in memory; `streams` is the normal case and is still++ allowed (the engine reads the rest from the SSD in place)."""++ if quant not in MAC_PACKS:++ raise ValueError(f"unknown Mac pack {quant!r}; one of {sorted(MAC_PACKS)}")++ pack = MAC_PACKS[quant]++ routed = float(pack["routed_gb"])++ dense = round(float(pack["backbone_gb"]) - routed, 2)++ cap = mac_expert_cap_gb(ram_gb or 0.0)++ resident = min(routed, cap)++ streaming = max(0.0, routed - resident)++ disk_need = float(pack["backbone_gb"]) + MAC_ENGRAM_GB + 6.0 # the shards, the Engram table, the pack folder++ out: dict[str, Any] = {"quant": quant, "name": pack["name"], "routed_gb": round(routed, 1), "dense_gb": dense,++ "resident_gb": round(resident, 1), "streaming_gb": round(streaming, 1),++ "engram_gb": MAC_ENGRAM_GB, "disk_need_gb": round(disk_need, 1),++ "disk_ok": None if disk_free_gb is None else float(disk_free_gb) >= disk_need,++ "ram_gb": ram_gb, "measured": MAC_MEASURED, "lines": [], "warnings": []}++ lines, warns = out["lines"], out["warnings"]++ lines.append(f"DeepSeek-V4.1-Flash {pack['name']}: {routed:.0f} GB of routed experts and {dense:.1f} GB of dense "++ "weights, plus a 111 GB Engram table that stays on the SSD and is read a few rows for each word.")++ if ram_gb and ram_gb >= 96:++ lines.append(f"This Mac ({ram_gb:.0f} GB): the engine will keep about {resident:.0f} GB of the experts in memory "++ f"and read the other {streaming:.0f} GB from the SSD while it answers.")++ elif ram_gb:++ lines.append(f"This Mac ({ram_gb:.0f} GB) can hold about {resident:.0f} GB of the experts; the other "++ f"{streaming:.0f} GB stream from the SSD, which is slow on a small Mac.")++ if cap <= 0:++ out["verdict"] = "small"++ warns.append("Too little memory to hold a useful part of the model: 64 GB or more is wanted.")++ elif streaming == 0:++ out["verdict"] = "fits"++ lines.append("Every routed expert stays in memory: the fastest case.")++ elif resident < 0.25 * routed:++ out["verdict"] = "small"++ warns.append(f"Only about {resident:.0f} GB of {routed:.0f} GB of experts fit in memory (under a quarter): "++ "most of the model is read from the SSD for every word. Expect seconds a word rather than tokens a second.")++ else:++ out["verdict"] = "streams"++ lines.append("The model streams: answers are slower than on a machine that holds all the experts, and they still "++ "work. An NVMe SSD is wanted, not a USB or network drive.")++ if disk_free_gb is not None and disk_free_gb < disk_need:++ warns.append(f"The files need about {disk_need:.0f} GB of free disk space and {disk_free_gb:.0f} GB are free.")++ lines.append("Measured on " + MAC_MEASURED + ".")++ out["words"] = " ".join(lines)++ return outdiff --cc tests/terminal/test_dsv41_fit_mac.pyindex 000000000,000000000..3f2e0cfb0new file mode 100644--- /dev/null+++ b/tests/terminal/test_dsv41_fit_mac.py@@@ -1,0 -1,0 +1,72 @@@++"""DeepSeek-V4.1-Flash on an Apple Silicon Mac: the fit numbers (`strata_control/dsv41_fit.py`, `mac_expert_cap_gb` and++`mac_plan`). The cap mirrors the engine's own rule, so 128 GiB must give the 71.4 GB of VRAM experts measured on the M5++Max on 2026-10-25. Pure: nothing here builds, downloads or starts anything."""++++from __future__ import annotations++++import unittest++++from strata_control import dsv41_fit as F++++++class MacExpertCap(unittest.TestCase):++ def test_the_m5_max_gives_the_measured_71_4_gb(self):++ self.assertAlmostEqual(F.mac_expert_cap_gb(128), 71.4, delta=0.3)++++ def test_a_32_gb_mac_mini_keeps_little(self):++ # 32 GiB: leave = 15% (4.8 GiB = 5.15 GB) and the working set caps it at 3/4 of 34.4 GB++ cap = F.mac_expert_cap_gb(32)++ self.assertGreater(cap, 0.0)++ self.assertLess(cap, 131.0 * 0.25) # well under a quarter of the 2.00 pack++ self.assertLess(F.mac_expert_cap_gb(16), F.mac_expert_cap_gb(32))++ self.assertEqual(F.mac_expert_cap_gb(0), 0.0)++++ def test_a_big_mac_stops_at_the_working_set(self):++ # 512 GiB: the file-cache floor rule leaves 40% and the device share is the tighter of the two? no: 40% wins++ cap = F.mac_expert_cap_gb(512)++ self.assertGreater(cap, 300.0) # far more than any pack needs++++++class MacPlan(unittest.TestCase):++ def test_the_default_pack_on_the_m5_max_streams(self):++ p = F.mac_plan(128, "IQ2_XS")++ self.assertEqual(p["verdict"], "streams")++ self.assertAlmostEqual(p["resident_gb"], 71.4, delta=0.3)++ self.assertAlmostEqual(p["streaming_gb"], 149.06 - 71.4, delta=0.3)++ self.assertEqual(p["dense_gb"], 6.17)++ self.assertIn("streams", p["words"])++ self.assertIn("M5 Max", p["words"])++++ def test_the_1_75_pack_streams_less_on_the_same_mac(self):++ small = F.mac_plan(128, "IQ2_XS")["streaming_gb"]++ self.assertLess(F.mac_plan(128, "IQ1_M")["streaming_gb"], small)++ self.assertGreater(F.mac_plan(128, "IQ2_XXS")["streaming_gb"], F.mac_plan(128, "IQ1_M")["streaming_gb"])++++ def test_a_mac_with_enough_memory_fits(self):++ p = F.mac_plan(512, "IQ2_XS")++ self.assertEqual(p["verdict"], "fits")++ self.assertEqual(p["streaming_gb"], 0.0)++++ def test_a_small_mac_is_told_plainly(self):++ p = F.mac_plan(32, "IQ2_XS")++ self.assertEqual(p["verdict"], "small")++ self.assertTrue(p["warnings"])++ self.assertTrue(any("read from the SSD" in w or "Expert" in w or "quarter" in w for w in p["warnings"]))++++ def test_disk_and_unknown_pack(self):++ p = F.mac_plan(128, "IQ2_XS", disk_free_gb=50.0)++ self.assertFalse(p["disk_ok"])++ self.assertTrue(any("disk space" in w for w in p["warnings"]))++ self.assertTrue(F.mac_plan(128, "IQ2_XS", disk_free_gb=900.0)["disk_ok"])++ with self.assertRaises(ValueError):++ F.mac_plan(128, "IQ9_Z")++++ def test_every_pack_is_described(self):++ for q in F.MAC_PACKS:++ p = F.mac_plan(128, q)++ self.assertTrue(p["words"])++ self.assertGreater(p["disk_need_gb"], F.MAC_ENGRAM_GB)++++++if __name__ == "__main__":++ unittest.main()diff --cc third_party/ggml/ggml-common.hindex 1dbbe326d,1dbbe326d..ca283d18e--- a/third_party/ggml/ggml-common.h+++ b/third_party/ggml/ggml-common.h@@@ -1131,7 -1131,7 +1131,7 @@@ GGML_TABLE_END(#define NGRID_IQ1S 2048#define IQ1S_DELTA 0.125f#define IQ1M_DELTA 0.125f--#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP)++#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP) || defined(GGML_COMMON_IMPL_METAL) // (Strata: ds4_iq.metal's IQ1_M matvec and moe_mm.metal's tile read iq1s_grid)GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,