Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: the hyper-connection family on the Mac - hc_post, hc_mix_scale, hc_mix ported
Niko12217022bd25d802
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.hindex f43f3f1ec..c9c47fe39 100644--- a/metal/include/metal_compat/ds4_metal_host.h+++ b/metal/include/metal_compat/ds4_metal_host.h@@ -32,6 +32,15 @@ struct FinishIn {uint8_t* pdst;long long pdst_stride;};+// ds4_kernels.cu:1901 hc_mix+bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes);++// ds4_kernels.cu:1692 hc_mix_scale+bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes);++// ds4_kernels.cu:1677 hc_post_inplace+bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T);+bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f);bool sgemm_tn(const float* A, const float* B, float* C, int m, int nt, int k, int lda, int ldb, int ldc);diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cppindex 84bd691b6..654bb9738 100644--- a/metal/src/engine/ds4_finish.cpp+++ b/metal/src/engine/ds4_finish.cpp@@ -102,4 +102,100 @@ bool finish(float* x, int n_vec, int len, long long stride, const FinishIn& f) {return true;}++// ds4_kernels.cu:1677 - hc_post_inplace: h[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*h_old[t][j][e]+bool hc_post(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T) {+ if (!ensure()) return false;+ void* hy = strata::gpu::host_view((void*) y);+ void* hh = strata::gpu::host_view(h);+ if (!hy || !hh) { std::fprintf(stderr, "dsv41 metal: hc_post has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer by = wrap(hy, (size_t) T * dim * 4 + 65536);+ mtl::Buffer bh = wrap(hh, (size_t) T * hc * dim * 4 + 65536);+ if (!by.handle || !bh.handle) return false;+ // post is [T][8], comb is [T][64], straight from the caller's host/shared memory+ mtl::Buffer bp = g_cx->alloc((size_t) T * 8 * 4);+ mtl::Buffer bc = g_cx->alloc((size_t) T * 64 * 4);+ if (!bp.ptr || !bc.ptr) return false;+ std::memcpy(bp.ptr, post, (size_t) T * 8 * 4);+ std::memcpy(bc.ptr, comb, (size_t) T * 64 * 4);+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_post"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(by, off(hy)), mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bp), mtl::Arg::B(bc),+ mtl::Arg::V((unsigned) dim), mtl::Arg::V((unsigned) hc)};+ d.grid[0] = (uint32_t) ((dim + 255) / 256);+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ g_cx->free(bp);+ g_cx->free(bc);+ return true;+}+++// ds4_kernels.cu:1692 - hc_mix_scale: mixes[t][i] *= rsqrt(sum(h[t])^2 / hcdim + eps)+bool hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes) {+ if (!ensure()) return false;+ void* hh = strata::gpu::host_view((void*) h);+ void* hm = strata::gpu::host_view((void*) mixes);+ if (!hh || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix_scale has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 + 65536);+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 + 65536);+ if (!bh.handle || !bm.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix_scale"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bm, off(hm)), mtl::Arg::V((unsigned) hcdim),+ mtl::Arg::V((unsigned) n_mix), mtl::Arg::V(eps)};+ d.grid[0] = (uint32_t) T;+ d.grid[1] = d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+++// ds4_kernels.cu:1901 - hc_mix: mixes[t][m] = (h[t] . fn[m]) * rsqrt(mean(h[t]^2) + eps)+bool hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes) {+ if (!ensure()) return false;+ void* hh = strata::gpu::host_view((void*) h);+ void* hf = strata::gpu::host_view((void*) fn);+ void* hm = strata::gpu::host_view((void*) mixes);+ if (!hh || !hf || !hm) { std::fprintf(stderr, "dsv41 metal: hc_mix has no host view\n"); return false; }+ const size_t page = 16384;+ auto wrap = [&](void* host, size_t bytes) {+ return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);+ };+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };+ mtl::Buffer bh = wrap(hh, (size_t) T * hcdim * 4 + 65536);+ mtl::Buffer bf = wrap(hf, (size_t) n_mix * hcdim * 4 + 65536);+ mtl::Buffer bm = wrap(hm, (size_t) T * 32 * 4 + 65536);+ if (!bh.handle || !bf.handle || !bm.handle) return false;+ mtl::Pipeline pl;+ try { pl = g_cx->pipeline(g_lib, "ds4_hc_mix"); } catch (...) { return false; }+ mtl::Dispatch d;+ d.p = &pl;+ d.args = {mtl::Arg::B(bh, off(hh)), mtl::Arg::B(bf, off(hf)), mtl::Arg::B(bm, off(hm)),+ mtl::Arg::V((unsigned) hcdim), mtl::Arg::V(eps)};+ d.grid[0] = (uint32_t) n_mix;+ d.grid[1] = (uint32_t) T;+ d.grid[2] = 1;+ d.tg[0] = 256; d.tg[1] = d.tg[2] = 1;+ g_cx->run({d});+ return true;+}+} // namespace strata::dsv41::metal_hostdiff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metalindex 03e81bf15..e0013e729 100644--- a/metal/src/engine/msl/ds4_init.metal+++ b/metal/src/engine/msl/ds4_init.metal@@ -113,3 +113,63 @@ kernel void ds4_finish(device float* x [[buffer(0)]], constant long& stride [[bu}}}++// ds4_kernels.cu:1677 hc_post_inplace_kernel: h[t][k][e] = post[t][k]*y[t][e] + sum_j comb[t][j][k]*h_old[t][j][e]+kernel void ds4_hc_post(device const float* y [[buffer(0)]], device float* h [[buffer(1)]],+ device const float* post [[buffer(2)]], device const float* comb [[buffer(3)]],+ constant uint& dim [[buffer(4)]], constant uint& hc [[buffer(5)]],+ uint2 tgp [[threadgroup_position_in_grid]], ushort lane [[thread_index_in_threadgroup]]) {+ const uint e = tgp.x * 256u + uint(lane);+ if (e >= dim) return;+ const uint t = tgp.y;+ device float* r = h + (size_t) t * hc * dim;+ float rv[8];+ for (uint j = 0; j < 8u; ++j) if (j < hc) rv[j] = r[(size_t) j * dim + e];+ const float yv = y[(size_t) t * dim + e];+ for (uint k = 0; k < 8u; ++k) {+ if (k >= hc) break;+ float a = post[(size_t) t * 8u + k] * yv;+ for (uint j = 0; j < 8u; ++j) if (j < hc) a += comb[(size_t) t * 64u + j * hc + k] * rv[j];+ r[(size_t) k * dim + e] = a;+ }+}++// ds4_kernels.cu:1692 hc_mix_scale_kernel: mixes[t][i] *= rsqrt(sum(h[t])^2 / hcdim + eps), one threadgroup a token+kernel void ds4_hc_mix_scale(device const float* h [[buffer(0)]], device float* mixes [[buffer(1)]],+ constant uint& hcdim [[buffer(2)]], constant uint& n_mix [[buffer(3)]],+ constant float& eps [[buffer(4)]], uint b [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[8];+ const device float* x = h + (size_t) b * hcdim;+ float ss = 0.0f;+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {+ const float4 hv = *(const device float4*) (x + k);+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;+ }+ ss = ds4_block_sum(ss, red, lane, w);+ const float inv = rsqrt(ss / float(hcdim) + eps);+ if (tid < n_mix) mixes[(size_t) b * 32u + tid] *= inv;+}++// ds4_kernels.cu:1017 hc_mix_kernel: mixes[t][m] = (h[t] . fn[m]) * rsqrt(mean(h[t]^2) + eps), one threadgroup a (m, t)+kernel void ds4_hc_mix(device const float* h [[buffer(0)]], device const float* fn [[buffer(1)]],+ device float* mixes [[buffer(2)]], constant uint& hcdim [[buffer(3)]],+ constant float& eps [[buffer(4)]], uint2 tgp [[threadgroup_position_in_grid]],+ uint tid [[thread_index_in_threadgroup]], ushort lane [[thread_index_in_simdgroup]],+ ushort w [[simdgroup_index_in_threadgroup]]) {+ threadgroup float red[8];+ const uint m = tgp.x, t = tgp.y;+ const device float* x = h + (size_t) t * hcdim;+ const device float* f = fn + (size_t) m * hcdim;+ float acc = 0.0f, ss = 0.0f;+ for (uint k = tid * 4u; k < hcdim; k += 256u * 4u) {+ const float4 hv = *(const device float4*) (x + k);+ const float4 fv = *(const device float4*) (f + k);+ acc += hv.x * fv.x + hv.y * fv.y + hv.z * fv.z + hv.w * fv.w;+ ss += hv.x * hv.x + hv.y * hv.y + hv.z * hv.z + hv.w * hv.w;+ }+ acc = ds4_block_sum(acc, red, lane, w);+ ss = ds4_block_sum(ss, red, lane, w);+ if (tid == 0) mixes[(size_t) t * 32u + m] = acc * rsqrt(ss / float(hcdim) + eps);+}diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex 8453d4f9f..342ed98e1 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1898,6 +1898,9 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cud}void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::hc_mix(h, fn, hcdim, n_mix, T, eps, mixes)) return;+#endifstrata_ds4_launch(hc_mix_kernel, dim3(n_mix, T), kThreads, 0, s, h, fn, hcdim, eps, mixes);}void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,@@ -1915,9 +1918,15 @@ void hc_post(const float* y, const float* res, const float* post, const float* cvoid hc_post_inplace(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T,cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::hc_post(y, h, post, comb, dim, hc, T)) return;+#endifstrata_ds4_launch(hc_post_inplace_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, y, h, post, comb, dim, hc);}void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {+#if defined(STRATA_METAL)+ if (strata::dsv41::metal_host::hc_mix_scale(h, hcdim, n_mix, T, eps, mixes)) return;+#endifstrata_ds4_launch(hc_mix_scale_kernel, T, kThreads, 0, s, h, hcdim, n_mix, eps, mixes);}void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, int fmt) {