Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: decode matvecs for Q5_K and IQ2_XXS (every lane busy, 16-byte loads), lane-parallel hc_split for hc 4, DS4_PROF per-kernel GPU table
Niko12213e51b1c18a32
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 143cef046..7da666011 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -307,7 +307,7 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,mtl::Buffer bb = wrap(hb, nbase * sizeof(float));if (!bs.handle || !bb.handle) return false;mtl::Pipeline pl;- try { pl = g_cx->pipeline(g_lib, "ds4_hc_split"); } catch (...) { return false; }+ try { pl = g_cx->pipeline(g_lib, hc == 4 && !std::getenv("DS4_NO_HCS4") ? "ds4_hc_split4" : "ds4_hc_split"); } catch (...) { return false; }mtl::Dispatch d;d.p = &pl;d.args = {mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bp, off(hp)),@@ -584,7 +584,7 @@ bool iq_mv(const void* jobs, int nj) {if (nj <= 0) return true;if (!ensure()) return false;const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;- std::vector<Ds4IqMvJob> exp_iq, exp_k;+ std::vector<Ds4IqMvJob> exp_iq, exp_k, exp_q5, exp_x1;exp_iq.reserve((size_t) nj);exp_k.reserve((size_t) nj);int max_n = 0;@@ -602,7 +602,10 @@ bool iq_mv(const void* jobs, int nj) {b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);b.y = q.y + (size_t) c0 * (size_t) q.ycs;}- (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);+ static const bool fast1 = std::getenv("DS4_NO_MV1") == nullptr;+ if (fast1 && b.ncol == 1 && b.type == 13) exp_q5.push_back(b); // the decode step's one-activation kernels+ else if (fast1 && b.ncol == 1 && b.type == 16) exp_x1.push_back(b);+ else (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);}}if (max_n <= 0) return true;@@ -638,6 +641,8 @@ bool iq_mv(const void* jobs, int nj) {g_cx->free(bj);return true;};+ if (!run_group(exp_q5, "ds4_q5k_mv1")) return false;+ if (!run_group(exp_x1, "ds4_iq2xxs_mv1")) return false;if (!run_group(exp_k, "ds4_iq_mv_k")) return false;if (!run_group(exp_iq, "ds4_iq_mv")) return false;return true;diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex 665087cb7..20c60ed24 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -2022,3 +2022,39 @@ kernel void ds4_args_probe(device float* out [[buffer(0)]], constant uint& dim [constant uint& hc [[buffer(2)]], uint gid [[thread_position_in_grid]]) {if (gid == 0) { out[0] = float(dim); out[1] = float(hc); out[2] = 1.0f; }}++// hc == 4 (every layer of this model): the warp-lane form of the CUDA kernel - lane j*4+k holds comb[j][k], the row and column+// sums are xor-shuffles (1, 2 within a row; 4, 8 within a column). One simdgroup a token instead of one lane.+kernel void ds4_hc_split4(device const float* mixes [[buffer(0)]], device const float* scale [[buffer(1)]],+ device const float* base [[buffer(2)]], device float* pre [[buffer(3)]],+ device float* post [[buffer(4)]], device float* comb [[buffer(5)]],+ constant uint& hc [[buffer(6)]], constant uint& iters [[buffer(7)]],+ constant float& eps [[buffer(8)]], uint b [[threadgroup_position_in_grid]],+ uint lane [[thread_index_in_threadgroup]]) {+ const device float* mx = mixes + (size_t) b * 32u;+ if (lane < 4u) {+ pre[(size_t) b * 8u + lane] = 1.0f / (1.0f + exp(-(mx[lane] * scale[0] + base[lane]))) + eps;+ post[(size_t) b * 8u + lane] = 2.0f / (1.0f + exp(-(mx[4u + lane] * scale[1] + base[4u + lane])));+ }+ const bool on = lane < 16u;+ float c = on ? mx[8u + lane] * scale[2] + base[8u + lane] : -3.402823466e+38f;+ float m = fmax(c, simd_shuffle_xor(c, ushort(1)));+ m = fmax(m, simd_shuffle_xor(m, ushort(2)));+ c = on ? exp(c - m) : 0.0f;+ float rs = c + simd_shuffle_xor(c, ushort(1));+ rs += simd_shuffle_xor(rs, ushort(2));+ c = c / rs + eps;+ c = on ? c : 0.0f;+ float cs = c + simd_shuffle_xor(c, ushort(4));+ cs += simd_shuffle_xor(cs, ushort(8));+ c = c / (cs + eps);+ for (uint it = 1; it < iters; ++it) {+ float r = c + simd_shuffle_xor(c, ushort(1));+ r += simd_shuffle_xor(r, ushort(2));+ c = c / (r + eps);+ float s = c + simd_shuffle_xor(c, ushort(4));+ s += simd_shuffle_xor(s, ushort(8));+ c = c / (s + eps);+ }+ if (on) comb[(size_t) b * 64u + lane] = c;+}diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metalindex 4bbea44f0..493e5aff1 100644--- a/metal/src/engine/msl/ds4_iq.metal+++ b/metal/src/engine/msl/ds4_iq.metal@@ -656,3 +656,93 @@ kernel void ds4_mx_dn(constant Ds4MxJob& j [[buffer(0)]], uint2 tgp [[threadgrou}}}++// ---------------------------------------------------------------------------------------------------------------------+// One-activation matvecs (the decode step): every lane of the row's simdgroup works, on a 32-value sub-block (IQ2_XXS) or+// a 64-value pair (Q5_K), where the 16-column kernels above give a whole 256-value block to a lane (a k = 1280 row used 5+// lanes of 32) and read it a byte at a time. Same results up to the order of the float sums.++kernel void ds4_q5k_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint j = lane & 3u; // this lane's pair of sub-blocks: 2j and 2j+1 (one 32-byte run of qs, both nibbles)+ const uint bsel = lane >> 2; // eight blocks a pass+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 8u) {+ const device uchar* p = w + (size_t) blk * 176u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));+ int sc0, mm0, sc1, mm1;+ ds4_k_scale(p + 4, int(2u * j), sc0, mm0);+ ds4_k_scale(p + 4, int(2u * j + 1u), sc1, mm1);+ const device uint4* qs4 = (const device uint4*) (p + 48u + 32u * j);+ const device uint4* qh4 = (const device uint4*) (p + 16);+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 64u * j);+ const uint4 qa = qs4[0], qb = qs4[1], ha = qh4[0], hb = qh4[1];+ const uint4 xa = xq4[0], xb = xq4[1], xc = xq4[2], xd4 = xq4[3]; // sub-block 2j: xa xb; 2j+1: xc xd4+ const uint qsw[8] = {qa.x, qa.y, qa.z, qa.w, qb.x, qb.y, qb.z, qb.w};+ const uint qhw[8] = {ha.x, ha.y, ha.z, ha.w, hb.x, hb.y, hb.z, hb.w};+ const uint xlo[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ const uint xhi[8] = {xc.x, xc.y, xc.z, xc.w, xd4.x, xd4.y, xd4.z, xd4.w};+ int sq0 = 0, sx0 = 0, sq1 = 0, sx1 = 0;+ for (uint t = 0; t < 8u; ++t) {+ const uint q0 = (qsw[t] & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j)) & 0x01010101u) << 4);+ const uint q1 = ((qsw[t] >> 4) & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j + 1u)) & 0x01010101u) << 4);+ for (uint i = 0; i < 4u; ++i) {+ const int x0 = int(char(xlo[t] >> (8u * i))), x1 = int(char(xhi[t] >> (8u * i)));+ sq0 += int((q0 >> (8u * i)) & 0xffu) * x0;+ sq1 += int((q1 >> (8u * i)) & 0xffu) * x1;+ sx0 += x0;+ sx1 += x1;+ }+ }+ acc = fma(b.xd[blk * 8u + 2u * j], d * float(sc0) * float(sq0) - dmin * float(mm0) * float(sx0), acc);+ acc = fma(b.xd[blk * 8u + 2u * j + 1u], d * float(sc1) * float(sq1) - dmin * float(mm1) * float(sx1), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+}++kernel void ds4_iq2xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));+ const uint row = tgp.x * 8u + (tid >> 5);+ if (int(row) >= b.n) return;+ const uint nblk = uint(b.k) / 256u;+ const device uchar* w = b.w + (size_t) row * b.ld;+ const uint ib = lane & 7u; // this lane's 32-value sub-block+ const uint bsel = lane >> 3; // four blocks a pass+ float acc = 0.0f;+ for (uint blk = bsel; blk < nblk; blk += 4u) {+ const device uchar* p = w + (size_t) blk * 66u;+ const float d = float(as_type<half>(*(const device ushort*) p));+ const device ushort* us = (const device ushort*) (p + 2u + 8u * ib); // (2-byte aligned: four 16-bit loads)+ const uint aux0 = uint(us[0]) | (uint(us[1]) << 16);+ const uint aux1 = uint(us[2]) | (uint(us[3]) << 16);+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);+ const uint4 xa = xq4[0], xb = xq4[1];+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};+ int sum = 0;+ for (uint l = 0; l < 4u; ++l) {+ const ulong g = iq2xxs_grid[(aux0 >> (8u * l)) & 0xffu];+ const uint sg = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);+ for (uint h = 0; h < 2u; ++h) {+ const uint gw = uint(g >> (32u * h));+ const uint x = xw[2u * l + h];+ for (uint i = 0; i < 4u; ++i) {+ const int code = int((gw >> (8u * i)) & 0xffu);+ const int xv = int(char(x >> (8u * i)));+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;+ }+ }+ }+ acc = fma(b.xd[blk * 8u + ib] * db, float(sum), acc);+ }+ const float v = simd_sum(acc);+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);+}diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex 440d4fcd0..d6c2070f5 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -318,12 +318,26 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {}static double g_gpu_ms = 0.0; // GPU time of every committed command buffer, summed+static std::mutex g_prof_mu;+static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)+static std::string g_prof_last;static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded, and command buffers committedstatic std::atomic<uint64_t> g_cb_count{0};static bool g_count_reg = false;static void ds4_report_counts() {std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,(unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());+ if (!g_prof.empty()) {+ std::vector<std::pair<double, std::string>> pv;+ double tot = 0;+ for (auto& kv : g_prof) { pv.emplace_back(kv.second.first, kv.first); tot += kv.second.first; }+ std::sort(pv.begin(), pv.end(), [](auto& a, auto& b) { return a.first > b.first; });+ std::fprintf(stderr, "prof: kernel GPU ms n us/dispatch (total %.1f ms)\n", tot / 1000.0);+ for (size_t i = 0; i < pv.size() && i < 30; ++i) {+ auto& e = g_prof[pv[i].second];+ std::fprintf(stderr, "prof: %-30s %8.1f %6llu %8.1f\n", pv[i].second.c_str(), e.first / 1000.0, (unsigned long long) e.second, e.first / (double) e.second);+ }+ }std::vector<std::pair<uint64_t, std::string>> v;{ std::lock_guard<std::mutex> lk(g_hist_mu); for (auto& kv : g_kernel_counts) v.emplace_back(kv.second, kv.first); }std::sort(v.begin(), v.end(), [](auto& a, auto& b) { return a.first > b.first; });@@ -367,7 +381,8 @@ void Context::flush_runs() {}if (impl_->inflight != nil) {[impl_->inflight waitUntilCompleted];- { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0; }+ { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;+ if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }if (impl_->inflight.status != MTLCommandBufferStatusCompleted)std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);impl_->inflight = nil;@@ -400,6 +415,8 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,}++impl_->def_n;++g_disp_count;+ static const bool prof = std::getenv("DS4_PROF") != nullptr; // every dispatch alone, its GPU time kept per kernel+ if (prof) { g_prof_last = ds[0].p->name; if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak(); flush_runs(); return 0.0; }static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr; // debugging: every dispatch alone, in orderstatic const std::string only = std::getenv("DS4_FLUSH_AFTER") ? std::getenv("DS4_FLUSH_AFTER") : ""; // ",name1,name2,"const bool listed = !only.empty() && only.find("," + ds[0].p->name + ",") != std::string::npos;