Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: decode matvecs for Q5_K and IQ2_XXS (every lane busy, 16-byte loads), lane-parallel hc_split for hc 4, DS4_PROF per-kernel GPU table

Niko12213e51b1c18a32
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 143cef046..7da666011 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -307,7 +307,7 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,
mtl::Buffer bb = wrap(hb, nbase * sizeof(float));
if (!bs.handle || !bb.handle) return false;
mtl::Pipeline pl;
- try { pl = g_cx->pipeline(g_lib, "ds4_hc_split"); } catch (...) { return false; }
+ try { pl = g_cx->pipeline(g_lib, hc == 4 && !std::getenv("DS4_NO_HCS4") ? "ds4_hc_split4" : "ds4_hc_split"); } catch (...) { return false; }
mtl::Dispatch d;
d.p = &pl;
d.args = {mtl::Arg::B(bm, off(hm)), mtl::Arg::B(bs, off(hs)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(bp, off(hp)),
@@ -584,7 +584,7 @@ bool iq_mv(const void* jobs, int nj) {
if (nj <= 0) return true;
if (!ensure()) return false;
const Ds4IqMvJob* src = (const Ds4IqMvJob*) jobs;
- std::vector<Ds4IqMvJob> exp_iq, exp_k;
+ std::vector<Ds4IqMvJob> exp_iq, exp_k, exp_q5, exp_x1;
exp_iq.reserve((size_t) nj);
exp_k.reserve((size_t) nj);
int max_n = 0;
@@ -602,7 +602,10 @@ bool iq_mv(const void* jobs, int nj) {
b.xd = q.xd + (size_t) c0 * (size_t) (q.k / 32);
b.y = q.y + (size_t) c0 * (size_t) q.ycs;
}
- (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);
+ static const bool fast1 = std::getenv("DS4_NO_MV1") == nullptr;
+ if (fast1 && b.ncol == 1 && b.type == 13) exp_q5.push_back(b); // the decode step's one-activation kernels
+ else if (fast1 && b.ncol == 1 && b.type == 16) exp_x1.push_back(b);
+ else (b.type == 13 || b.type == 14 ? exp_k : exp_iq).push_back(b);
}
}
if (max_n <= 0) return true;
@@ -638,6 +641,8 @@ bool iq_mv(const void* jobs, int nj) {
g_cx->free(bj);
return true;
};
+ if (!run_group(exp_q5, "ds4_q5k_mv1")) return false;
+ if (!run_group(exp_x1, "ds4_iq2xxs_mv1")) return false;
if (!run_group(exp_k, "ds4_iq_mv_k")) return false;
if (!run_group(exp_iq, "ds4_iq_mv")) return false;
return true;
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index 665087cb7..20c60ed24 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -2022,3 +2022,39 @@ kernel void ds4_args_probe(device float* out [[buffer(0)]], constant uint& dim [
constant uint& hc [[buffer(2)]], uint gid [[thread_position_in_grid]]) {
if (gid == 0) { out[0] = float(dim); out[1] = float(hc); out[2] = 1.0f; }
}
+
+// hc == 4 (every layer of this model): the warp-lane form of the CUDA kernel - lane j*4+k holds comb[j][k], the row and column
+// sums are xor-shuffles (1, 2 within a row; 4, 8 within a column). One simdgroup a token instead of one lane.
+kernel void ds4_hc_split4(device const float* mixes [[buffer(0)]], device const float* scale [[buffer(1)]],
+ device const float* base [[buffer(2)]], device float* pre [[buffer(3)]],
+ device float* post [[buffer(4)]], device float* comb [[buffer(5)]],
+ constant uint& hc [[buffer(6)]], constant uint& iters [[buffer(7)]],
+ constant float& eps [[buffer(8)]], uint b [[threadgroup_position_in_grid]],
+ uint lane [[thread_index_in_threadgroup]]) {
+ const device float* mx = mixes + (size_t) b * 32u;
+ if (lane < 4u) {
+ pre[(size_t) b * 8u + lane] = 1.0f / (1.0f + exp(-(mx[lane] * scale[0] + base[lane]))) + eps;
+ post[(size_t) b * 8u + lane] = 2.0f / (1.0f + exp(-(mx[4u + lane] * scale[1] + base[4u + lane])));
+ }
+ const bool on = lane < 16u;
+ float c = on ? mx[8u + lane] * scale[2] + base[8u + lane] : -3.402823466e+38f;
+ float m = fmax(c, simd_shuffle_xor(c, ushort(1)));
+ m = fmax(m, simd_shuffle_xor(m, ushort(2)));
+ c = on ? exp(c - m) : 0.0f;
+ float rs = c + simd_shuffle_xor(c, ushort(1));
+ rs += simd_shuffle_xor(rs, ushort(2));
+ c = c / rs + eps;
+ c = on ? c : 0.0f;
+ float cs = c + simd_shuffle_xor(c, ushort(4));
+ cs += simd_shuffle_xor(cs, ushort(8));
+ c = c / (cs + eps);
+ for (uint it = 1; it < iters; ++it) {
+ float r = c + simd_shuffle_xor(c, ushort(1));
+ r += simd_shuffle_xor(r, ushort(2));
+ c = c / (r + eps);
+ float s = c + simd_shuffle_xor(c, ushort(4));
+ s += simd_shuffle_xor(s, ushort(8));
+ c = c / (s + eps);
+ }
+ if (on) comb[(size_t) b * 64u + lane] = c;
+}
diff --git a/metal/src/engine/msl/ds4_iq.metal b/metal/src/engine/msl/ds4_iq.metal
index 4bbea44f0..493e5aff1 100644
--- a/metal/src/engine/msl/ds4_iq.metal
+++ b/metal/src/engine/msl/ds4_iq.metal
@@ -656,3 +656,93 @@ kernel void ds4_mx_dn(constant Ds4MxJob& j [[buffer(0)]], uint2 tgp [[threadgrou
}
}
}
+
+// ---------------------------------------------------------------------------------------------------------------------
+// One-activation matvecs (the decode step): every lane of the row's simdgroup works, on a 32-value sub-block (IQ2_XXS) or
+// a 64-value pair (Q5_K), where the 16-column kernels above give a whole 256-value block to a lane (a k = 1280 row used 5
+// lanes of 32) and read it a byte at a time. Same results up to the order of the float sums.
+
+kernel void ds4_q5k_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint j = lane & 3u; // this lane's pair of sub-blocks: 2j and 2j+1 (one 32-byte run of qs, both nibbles)
+ const uint bsel = lane >> 2; // eight blocks a pass
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 8u) {
+ const device uchar* p = w + (size_t) blk * 176u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const float dmin = float(as_type<half>(*(const device ushort*) (p + 2)));
+ int sc0, mm0, sc1, mm1;
+ ds4_k_scale(p + 4, int(2u * j), sc0, mm0);
+ ds4_k_scale(p + 4, int(2u * j + 1u), sc1, mm1);
+ const device uint4* qs4 = (const device uint4*) (p + 48u + 32u * j);
+ const device uint4* qh4 = (const device uint4*) (p + 16);
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 64u * j);
+ const uint4 qa = qs4[0], qb = qs4[1], ha = qh4[0], hb = qh4[1];
+ const uint4 xa = xq4[0], xb = xq4[1], xc = xq4[2], xd4 = xq4[3]; // sub-block 2j: xa xb; 2j+1: xc xd4
+ const uint qsw[8] = {qa.x, qa.y, qa.z, qa.w, qb.x, qb.y, qb.z, qb.w};
+ const uint qhw[8] = {ha.x, ha.y, ha.z, ha.w, hb.x, hb.y, hb.z, hb.w};
+ const uint xlo[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ const uint xhi[8] = {xc.x, xc.y, xc.z, xc.w, xd4.x, xd4.y, xd4.z, xd4.w};
+ int sq0 = 0, sx0 = 0, sq1 = 0, sx1 = 0;
+ for (uint t = 0; t < 8u; ++t) {
+ const uint q0 = (qsw[t] & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j)) & 0x01010101u) << 4);
+ const uint q1 = ((qsw[t] >> 4) & 0x0f0f0f0fu) | (((qhw[t] >> (2u * j + 1u)) & 0x01010101u) << 4);
+ for (uint i = 0; i < 4u; ++i) {
+ const int x0 = int(char(xlo[t] >> (8u * i))), x1 = int(char(xhi[t] >> (8u * i)));
+ sq0 += int((q0 >> (8u * i)) & 0xffu) * x0;
+ sq1 += int((q1 >> (8u * i)) & 0xffu) * x1;
+ sx0 += x0;
+ sx1 += x1;
+ }
+ }
+ acc = fma(b.xd[blk * 8u + 2u * j], d * float(sc0) * float(sq0) - dmin * float(mm0) * float(sx0), acc);
+ acc = fma(b.xd[blk * 8u + 2u * j + 1u], d * float(sc1) * float(sq1) - dmin * float(mm1) * float(sx1), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+}
+
+kernel void ds4_iq2xxs_mv1(device const uchar* jobs [[buffer(0)]], uint2 tgp [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]]) {
+ const device Ds4MvJob& b = *(const device Ds4MvJob*) (jobs + (size_t) tgp.y * sizeof(Ds4MvJob));
+ const uint row = tgp.x * 8u + (tid >> 5);
+ if (int(row) >= b.n) return;
+ const uint nblk = uint(b.k) / 256u;
+ const device uchar* w = b.w + (size_t) row * b.ld;
+ const uint ib = lane & 7u; // this lane's 32-value sub-block
+ const uint bsel = lane >> 3; // four blocks a pass
+ float acc = 0.0f;
+ for (uint blk = bsel; blk < nblk; blk += 4u) {
+ const device uchar* p = w + (size_t) blk * 66u;
+ const float d = float(as_type<half>(*(const device ushort*) p));
+ const device ushort* us = (const device ushort*) (p + 2u + 8u * ib); // (2-byte aligned: four 16-bit loads)
+ const uint aux0 = uint(us[0]) | (uint(us[1]) << 16);
+ const uint aux1 = uint(us[2]) | (uint(us[3]) << 16);
+ const float db = d * (0.5f + float(aux1 >> 28)) * 0.25f;
+ const device uint4* xq4 = (const device uint4*) (b.xq + (size_t) blk * 256u + 32u * ib);
+ const uint4 xa = xq4[0], xb = xq4[1];
+ const uint xw[8] = {xa.x, xa.y, xa.z, xa.w, xb.x, xb.y, xb.z, xb.w};
+ int sum = 0;
+ for (uint l = 0; l < 4u; ++l) {
+ const ulong g = iq2xxs_grid[(aux0 >> (8u * l)) & 0xffu];
+ const uint sg = uint(ksigns_iq2xs[(aux1 >> (7u * l)) & 127u]);
+ for (uint h = 0; h < 2u; ++h) {
+ const uint gw = uint(g >> (32u * h));
+ const uint x = xw[2u * l + h];
+ for (uint i = 0; i < 4u; ++i) {
+ const int code = int((gw >> (8u * i)) & 0xffu);
+ const int xv = int(char(x >> (8u * i)));
+ sum += ((sg >> (4u * h + i)) & 1u) != 0u ? -code * xv : code * xv;
+ }
+ }
+ }
+ acc = fma(b.xd[blk * 8u + ib] * db, float(sum), acc);
+ }
+ const float v = simd_sum(acc);
+ if (lane == 0u) b.y[row] = b.alpha * v + (b.bias != nullptr ? b.bias[row] : 0.0f);
+}
diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mm
index 440d4fcd0..d6c2070f5 100644
--- a/metal/src/mtl_rt.mm
+++ b/metal/src/mtl_rt.mm
@@ -318,12 +318,26 @@ static void encode(id<MTLComputeCommandEncoder> enc, const Dispatch& d) {
}
static double g_gpu_ms = 0.0; // GPU time of every committed command buffer, summed
+static std::mutex g_prof_mu;
+static std::unordered_map<std::string, std::pair<double, uint64_t>> g_prof; // DS4_PROF: kernel -> (GPU us, dispatches)
+static std::string g_prof_last;
static std::atomic<uint64_t> g_disp_count{0}; // dispatches encoded, and command buffers committed
static std::atomic<uint64_t> g_cb_count{0};
static bool g_count_reg = false;
static void ds4_report_counts() {
std::fprintf(stderr, "mtl: %.1f ms GPU, %llu dispatches, %llu command buffers\n", g_gpu_ms,
(unsigned long long) g_disp_count.load(), (unsigned long long) g_cb_count.load());
+ if (!g_prof.empty()) {
+ std::vector<std::pair<double, std::string>> pv;
+ double tot = 0;
+ for (auto& kv : g_prof) { pv.emplace_back(kv.second.first, kv.first); tot += kv.second.first; }
+ std::sort(pv.begin(), pv.end(), [](auto& a, auto& b) { return a.first > b.first; });
+ std::fprintf(stderr, "prof: kernel GPU ms n us/dispatch (total %.1f ms)\n", tot / 1000.0);
+ for (size_t i = 0; i < pv.size() && i < 30; ++i) {
+ auto& e = g_prof[pv[i].second];
+ std::fprintf(stderr, "prof: %-30s %8.1f %6llu %8.1f\n", pv[i].second.c_str(), e.first / 1000.0, (unsigned long long) e.second, e.first / (double) e.second);
+ }
+ }
std::vector<std::pair<uint64_t, std::string>> v;
{ std::lock_guard<std::mutex> lk(g_hist_mu); for (auto& kv : g_kernel_counts) v.emplace_back(kv.second, kv.first); }
std::sort(v.begin(), v.end(), [](auto& a, auto& b) { return a.first > b.first; });
@@ -367,7 +381,8 @@ void Context::flush_runs() {
}
if (impl_->inflight != nil) {
[impl_->inflight waitUntilCompleted];
- { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0; }
+ { const double d = impl_->inflight.GPUEndTime - impl_->inflight.GPUStartTime; if (d > 0) g_gpu_ms += d * 1000.0;
+ if (!g_prof_last.empty()) { std::lock_guard<std::mutex> pl(g_prof_mu); auto& e = g_prof[g_prof_last]; e.first += d * 1e6; e.second += 1; g_prof_last.clear(); } }
if (impl_->inflight.status != MTLCommandBufferStatusCompleted)
std::fprintf(stderr, "mtl: deferred command buffer failed: %s\n", [[impl_->inflight.error localizedDescription] UTF8String]);
impl_->inflight = nil;
@@ -400,6 +415,8 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,
}
++impl_->def_n;
++g_disp_count;
+ static const bool prof = std::getenv("DS4_PROF") != nullptr; // every dispatch alone, its GPU time kept per kernel
+ if (prof) { g_prof_last = ds[0].p->name; if (ds4_gpu_sync_all_streams_weak) ds4_gpu_sync_all_streams_weak(); flush_runs(); return 0.0; }
static const bool each = std::getenv("DS4_FLUSH_EACH") != nullptr; // debugging: every dispatch alone, in order
static const std::string only = std::getenv("DS4_FLUSH_AFTER") ? std::getenv("DS4_FLUSH_AFTER") : ""; // ",name1,name2,"
const bool listed = !only.empty() && only.find("," + ds[0].p->name + ",") != std::string::npos;