Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: Q5_K dequant four values a step (bit-identical); DS4_SKIP_K and the timeline's overlap count as debug tools
Niko1221100a2374fbe2
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex 5e20d11bd..b78715d59 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -2164,10 +2164,13 @@ kernel void ds4_dq_q5k(device const uchar* W [[buffer(0)]], constant ulong& ld [const device uchar* qh = p + 16;const device uchar* qs = p + 48 + 32u * (ib >> 1);const uint sh = 4u * (ib & 1u);- device half* o = out + (ulong) row * k + (ulong) blk * 256ul + ib * 32u;- for (uint l = 0; l < 32u; ++l) {- const uint q = ((uint(qs[l]) >> sh) & 15u) | (((uint(qh[l]) >> ib) & 1u) << 4);- o[l] = half(d * float(sc) * float(q) - dmin * float(mm));+ device packed_half4* o = (device packed_half4*) (out + (ulong) row * k + (ulong) blk * 256ul + ib * 32u);+ const float ds = d * float(sc), dm = dmin * float(mm);+ for (uint l = 0; l < 32u; l += 4u) { // four values a step: 4-byte loads, 8-byte stores (the same arithmetic a value)+ const uint4 a = uint4(*(const device packed_uchar4*) (qs + l));+ const uint4 b = uint4(*(const device packed_uchar4*) (qh + l));+ const uint4 q = ((a >> sh) & 15u) | (((b >> ib) & 1u) << 4);+ o[l >> 2] = packed_half4(half4(float4(ds) * float4(q) - float4(dm)));}}diff --git a/metal/src/mtl_rt.mm b/metal/src/mtl_rt.mmindex eb766c397..4950477bd 100644--- a/metal/src/mtl_rt.mm+++ b/metal/src/mtl_rt.mm@@ -170,8 +170,8 @@ void tl_report(id<MTLDevice> dev) {struct Acc { double busy = 0, gap = 0, gap_big = 0; size_t n = 0, n_big = 0; };std::map<std::string, Acc> acc;double t_first = 0, t_last = 0, total_busy = 0, total_gap = 0, total_big = 0;- double prev_end = -1;- size_t used = 0;+ double prev_end = -1, ovl_us = 0;+ size_t used = 0, n_ovl = 0;for (size_t k = from; k < n; ++k) {if (g_tl.names[k].empty()) continue;const double st = (double) ts[2 * k].timestamp * ns_per_gpu * 1e-3, en = (double) ts[2 * k + 1].timestamp * ns_per_gpu * 1e-3; // us@@ -181,6 +181,7 @@ void tl_report(id<MTLDevice> dev) {a.busy += en - st;++a.n;total_busy += en - st;+ if (prev_end >= 0 && st < prev_end) { ++n_ovl; ovl_us += prev_end - st; }if (prev_end >= 0 && st >= prev_end) {const double gp = st - prev_end;a.gap += gp;@@ -196,6 +197,7 @@ void tl_report(id<MTLDevice> dev) {std::sort(v.begin(), v.end(), [](auto& x, auto& y) { return x.first > y.first; });std::fprintf(stderr, "timeline: %zu dispatches, wall %.1f ms, GPU busy %.1f ms, idle gaps %.1f ms (of it > 300 us: %.1f ms)\n", used,(t_last - t_first) / 1000.0, total_busy / 1000.0, total_gap / 1000.0, total_big / 1000.0);+ std::fprintf(stderr, "timeline: %zu dispatches started before the previous one ended (%.1f ms overlapped)\n", n_ovl, ovl_us / 1000.0);std::fprintf(stderr, "timeline: kernel n busy ms us/disp gap ms gap/disp big-gap ms (n)\n");for (size_t i = 0; i < v.size() && i < 40; ++i) {const Acc& a = acc[v[i].second];@@ -774,6 +776,8 @@ double Context::run(const std::vector<Dispatch>& ds, int reps, bool concurrent,std::lock_guard<std::recursive_mutex> lk_(impl_->mu);@autoreleasepool {if (ds.size() == 1 && reps == 1 && !barrier_between) {+ { static const std::string skipk = std::getenv("DS4_SKIP_K") ? std::string(",") + std::getenv("DS4_SKIP_K") + "," : std::string(); // debugging: drop these kernels (timing only)+ if (!skipk.empty() && skipk.find("," + ds[0].p->name + ",") != std::string::npos) return 0.0; }if (impl_->cb_def == nil || impl_->def_n >= 2000000) {if (impl_->cb_def != nil) split_runs(); // (a buffer committed earlier may still be running: the queue keeps the order)if (std::getenv("DS4_ERRINFO")) {