Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

metal: the routing trio ported ahead of its turn - moe_sort counts, bounds scan, fill

Niko1221154c2990edb2
diff --git a/metal/include/metal_compat/ds4_metal_host.h b/metal/include/metal_compat/ds4_metal_host.h
index fba08b9af..d26707802 100644
--- a/metal/include/metal_compat/ds4_metal_host.h
+++ b/metal/include/metal_compat/ds4_metal_host.h
@@ -32,6 +32,9 @@ struct FinishIn {
uint8_t* pdst;
long long pdst_stride;
};
+// ds4_kernels.cu:1938 moe_sort: counts, prefix, fill
+bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos);
+
// ds4_kernels.cu:1908 hc_split (the sinkhorn normalisation)
bool hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,
float* pre, float* post, float* comb);
diff --git a/metal/src/engine/ds4_finish.cpp b/metal/src/engine/ds4_finish.cpp
index 37a5fe52e..e0a01b7a0 100644
--- a/metal/src/engine/ds4_finish.cpp
+++ b/metal/src/engine/ds4_finish.cpp
@@ -240,4 +240,62 @@ bool hc_split(const float* mixes, const float* scale, const float* base, int hc,
return true;
}
+
+// ds4_kernels.cu:1938 - moe_sort: count each virtual slot's picks, prefix-sum the counts, fill in token order.
+// Three dispatches on one command buffer, so their order is the order of the calls (serial dispatch).
+bool moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos) {
+ if (!ensure()) return false;
+ void* hi = strata::gpu::host_view((void*) idx);
+ void* hv = strata::gpu::host_view((void*) vs);
+ void* hb = strata::gpu::host_view((void*) bounds);
+ void* hr = strata::gpu::host_view((void*) row_tok);
+ void* hp = strata::gpu::host_view((void*) pos);
+ if (!hi || !hv || !hb || !hr || !hp) { std::fprintf(stderr, "dsv41 metal: moe_sort has no host view\n"); return false; }
+ const size_t page = 16384;
+ auto wrap = [&](void* host, size_t bytes) {
+ return g_cx->wrap((void*) (((uintptr_t) host) & ~(uintptr_t) (page - 1)), ((bytes + page - 1) / page) * page);
+ };
+ auto off = [&](void* host) { return (size_t) ((uintptr_t) host - (((uintptr_t) host) & ~(uintptr_t) (page - 1))); };
+ const size_t t_bytes = (size_t) T * k * 4 + 65536;
+ mtl::Buffer bi = wrap(hi, t_bytes);
+ mtl::Buffer bv = wrap(hv, (size_t) n_vs * 4 + 65536);
+ mtl::Buffer bb = wrap(hb, (size_t) (n_vs + 2) * 4 + 65536);
+ mtl::Buffer br = wrap(hr, t_bytes);
+ mtl::Buffer bp = wrap(hp, t_bytes);
+ if (!bi.handle || !bv.handle || !bb.handle || !br.handle || !bp.handle) return false;
+ mtl::Pipeline pc, pf, ps;
+ try {
+ pc = g_cx->pipeline(g_lib, "ds4_moe_sort_count");
+ pf = g_cx->pipeline(g_lib, "ds4_moe_sort_fill");
+ ps = g_cx->pipeline(g_lib, "ds4_bounds_scan");
+ } catch (...) { return false; }
+ // 1. counts
+ mtl::Dispatch d1;
+ d1.p = &pc;
+ d1.args = {mtl::Arg::B(bi, off(hi)), mtl::Arg::V((unsigned) T), mtl::Arg::V((unsigned) k),
+ mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bb, off(hb))};
+ d1.grid[0] = (uint32_t) n_vs;
+ d1.grid[1] = d1.grid[2] = 1;
+ d1.tg[0] = 256; d1.tg[1] = d1.tg[2] = 1;
+ g_cx->run({d1});
+ // 2. the exclusive prefix over the slots
+ mtl::Dispatch d2;
+ d2.p = &ps;
+ d2.args = {mtl::Arg::B(bb, off(hb)), mtl::Arg::V((unsigned) n_vs)};
+ d2.grid[0] = 1;
+ d2.grid[1] = d2.grid[2] = 1;
+ d2.tg[0] = 32; d2.tg[1] = d2.tg[2] = 1;
+ g_cx->run({d2});
+ // 3. the fill
+ mtl::Dispatch d3;
+ d3.p = &pf;
+ d3.args = {mtl::Arg::B(bi, off(hi)), mtl::Arg::V((unsigned) T), mtl::Arg::V((unsigned) k),
+ mtl::Arg::B(bv, off(hv)), mtl::Arg::B(bb, off(hb)), mtl::Arg::B(br, off(hr)), mtl::Arg::B(bp, off(hp))};
+ d3.grid[0] = (uint32_t) n_vs;
+ d3.grid[1] = d3.grid[2] = 1;
+ d3.tg[0] = 256; d3.tg[1] = d3.tg[2] = 1;
+ g_cx->run({d3});
+ return true;
+}
+
} // namespace strata::dsv41::metal_host
diff --git a/metal/src/engine/msl/ds4_init.metal b/metal/src/engine/msl/ds4_init.metal
index 6e7094c90..c2550903d 100644
--- a/metal/src/engine/msl/ds4_init.metal
+++ b/metal/src/engine/msl/ds4_init.metal
@@ -239,3 +239,73 @@ kernel void ds4_hc_split(device const float* mixes [[buffer(0)]], device const f
for (uint k = 0; k < 8u; ++k) { if (k >= hc) break; cb[j * hc + k] = c[j][k]; }
}
}
+
+// ds4_kernels.cu:1938 moe_sort - count each virtual slot's picks, prefix-sum the counts, then fill in token order.
+// kPicks = 8 (ds4_kernels.cuh:19): idx is [T][8], so q -> (t = q/8, i = q%8).
+kernel void ds4_bounds_scan(device int* bounds [[buffer(0)]], constant uint& n_vs [[buffer(1)]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ if (tid != 0u) return;
+ bounds[0] = 0;
+ for (uint s = 0; s < n_vs; ++s) bounds[s + 1] += bounds[s];
+}
+
+// simd_ballot returns a simd_vote, which does not cast to an integer in this toolchain; the same counts come from the
+// prefix-sum and sum primitives, which give a lane its offset inside its simdgroup and the group its total.
+static inline int ds4_warp_prefix(bool hit) { return simd_prefix_exclusive_sum(hit ? 1 : 0); }
+static inline int ds4_warp_total(bool hit) { return simd_sum(hit ? 1 : 0); }
+
+kernel void ds4_moe_sort_count(device const int* idx [[buffer(0)]], constant uint& T [[buffer(1)]],
+ constant uint& kk [[buffer(2)]], device const int* vs [[buffer(3)]],
+ device int* bounds [[buffer(4)]], uint slot [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup int warp_n[8];
+ threadgroup int base_s;
+ const uint wp = tid / 32u, lane = tid % 32u;
+ if (tid == 0u) base_s = 0;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const uint n = T * kk;
+ for (uint q = tid; q < n; q += 256u) {
+ const uint t = q / kk, i = q - t * kk;
+ const bool hit = vs[idx[t * 8u + i]] == int(slot);
+ // one vote a warp: the running offset needs each warp's count, accumulated in slot order
+ const int wtot = ds4_warp_total(hit);
+ if ((tid % 32u) == 0u) warp_n[wp] = wtot;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ int total = 0;
+ for (uint w = 0; w < 8u; ++w) total += warp_n[w];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) base_s += total;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ if (tid == 0u) bounds[slot + 1] = base_s;
+}
+
+kernel void ds4_moe_sort_fill(device const int* idx [[buffer(0)]], constant uint& T [[buffer(1)]],
+ constant uint& kk [[buffer(2)]], device const int* vs [[buffer(3)]],
+ device int* bounds [[buffer(4)]], device int* row_tok [[buffer(5)]],
+ device int* pos [[buffer(6)]], uint slot [[threadgroup_position_in_grid]],
+ uint tid [[thread_index_in_threadgroup]]) {
+ threadgroup int warp_n[8];
+ threadgroup int base_s;
+ const uint wp = tid / 32u, lane = tid % 32u;
+ if (tid == 0u) base_s = bounds[slot];
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ const uint n = T * kk;
+ for (uint q = tid; q < n; q += 256u) {
+ const uint t = q / kk, i = q - t * kk;
+ const bool hit = vs[idx[t * 8u + i]] == int(slot);
+ const int wpre = ds4_warp_prefix(hit), wtot = ds4_warp_total(hit);
+ if ((tid % 32u) == 0u) warp_n[wp] = wtot;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ int before = 0, total = 0;
+ for (uint w = 0; w < 8u; ++w) { if (w < wp) before += warp_n[w]; total += warp_n[w]; }
+ if (hit) {
+ const int r = base_s + before + wpre;
+ row_tok[r] = int(t);
+ pos[t * 8u + i] = r;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if (tid == 0u) base_s += total;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+}
diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cu
index e387ae1f4..21343c8fd 100644
--- a/src/dsv41/ds4_kernels.cu
+++ b/src/dsv41/ds4_kernels.cu
@@ -1937,6 +1937,9 @@ void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd,
}
void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,
cudaStream_t s) {
+#if defined(STRATA_METAL)
+ if (strata::dsv41::metal_host::moe_sort(idx, T, k, vs, n_vs, bounds, row_tok, pos)) return;
+#endif
strata_ds4_launch(moe_sort_kernel<false>, n_vs, kThreads, 0, s, idx, T, k, vs, bounds, row_tok, pos);
strata_ds4_launch(bounds_scan_kernel, 1, 32, 0, s, bounds, n_vs);
strata_ds4_launch(moe_sort_kernel<true>, n_vs, kThreads, 0, s, idx, T, k, vs, bounds, row_tok, pos);