Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
metal: the dsv41 engine links and runs on the Mac
Niko1221883d0000a7c8
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txtindex 91b7e0326..f585cb4a7 100644--- a/metal/CMakeLists.txt+++ b/metal/CMakeLists.txt@@ -99,7 +99,8 @@ set(STRATA_METAL_HOST_SOURCES${S}/spec/suffix_drafter.cpp ${S}/spec/controller.cpp ${S}/spec/draft_policy.cpp ${S}/spec/draft_source.cpp${S}/kernels/cpu/pool.cpp ${S}/kernels/cpu/expert_layout.cpp ${S}/kernels/cpu/native_expert.cpp${S}/kernels/cpu/hadamard.cpp ${S}/kernels/cpu/tq_cpu.cpp) # Gyro: the Hadamard rotation, the scalar trellis rows-set_source_files_properties(${S}/core/device.cu ${S}/core/pinned.cu PROPERTIES LANGUAGE CXX)+set_source_files_properties(${S}/core/device.cu ${S}/core/pinned.cu ${DS4}/ds4_vision.cu ${DS4}/vendor/exl3.cu ${DS4}${DS4}/vendor/moe_win.cu ${DS4}/vendor/sampler.cu+ PROPERTIES LANGUAGE CXX)# The kernel library: Metal implementations (src/engine/*.cpp) and, for every entry point not ported yet, a stub# that stops the engine with the entry point's name (src/engine/stubs.cpp, generated by tools/gen_stubs.py).@@ -127,16 +128,22 @@ target_link_libraries(strata PRIVATE strata_metal_kernels strata_metal_host stra# through include/metal_compat, which gives them CUDA's names over the Metal runtime; the kernel entry points they call are# the Metal implementations in src/engine/* or the stubs from tools/gen_stubs.py that stop the engine with the name.set(DS4 ${STRATA_ROOT}/src/dsv41)+# The DeepSeek engine's sources. Every .cu compiles as C++ through include/metal_compat; the EXL3 legacy path is left out+# (the 2.25bpw text model never calls it) and the vision tower is included because ds4_model.cu references it.set(STRATA_DS4_SOURCES- ${DS4}/ds4_main.cpp- ${DS4}/ds4_model.cu ${DS4}/ds4_gguf.cu ${DS4}/ds4_iq.cu ${DS4}/ds4_kernels.cu ${DS4}/ds4_tq.cu+ ${DS4}/ds4_main.cpp ${DS4}/ds4_model.cu ${DS4}/ds4_gguf.cu ${DS4}/ds4_iq.cu ${DS4}/ds4_kernels.cu ${DS4}/ds4_tq.cu+ ${DS4}/ds4_vision.cu+ ${DS4}/vendor/exl3.cu ${DS4}/vendor/exl3_cpu.cpp ${DS4}/vendor/moe_win.cu ${DS4}/vendor/sampler.cu${DS4}/ds4_gg_cpu.cpp ${DS4}/ds4_gguf_cpu.cpp ${DS4}/ds4_iq_cpu.cpp ${DS4}/ds4_tq_cpu.cpp ${DS4}/ds4_packinfo.cpp)-set_source_files_properties(${DS4}/ds4_model.cu ${DS4}/ds4_gguf.cu ${DS4}/ds4_iq.cu ${DS4}/ds4_kernels.cu ${DS4}/ds4_tq.cu- PROPERTIES LANGUAGE CXX)+set_source_files_properties(+ ${DS4}/ds4_model.cu ${DS4}/ds4_gguf.cu ${DS4}/ds4_iq.cu ${DS4}/ds4_kernels.cu ${DS4}/ds4_tq.cu ${DS4}/ds4_vision.cu+ ${DS4}/vendor/exl3.cu ${DS4}/vendor/moe_win.cu ${DS4}/vendor/sampler.cu+ PROPERTIES LANGUAGE CXX)+list(APPEND STRATA_DS4_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/engine/exl3_legacy_stub.cpp) # v1.1.8 EXL3: absent on Metaladd_executable(strata_ds4 ${STRATA_DS4_SOURCES})target_include_directories(strata_ds4 PRIVATE- ${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat ${STRATA_ROOT}/include ${DS4}- ${STRATA_ROOT}/third_party/ggml ${STRATA_GGML_DIR}/ggml/include ${STRATA_GGML_DIR}/ggml/src)+ ${CMAKE_CURRENT_SOURCE_DIR}/include/metal_compat ${STRATA_ROOT}/include ${DS4} ${CMAKE_CURRENT_SOURCE_DIR}/src+ ${STRATA_ROOT}/third_party/ggml ${STRATA_GGML_DIR}/ggml/include ${STRATA_GGML_DIR}/ggml/src ${STRATA_GGML_DIR}/ggml/src/ggml-cpu)target_compile_definitions(strata_ds4 PRIVATE STRATA_METAL=1)target_compile_options(strata_ds4 PRIVATE -Wno-unused-parameter -Wno-deprecated-declarations -Wno-implicit-function-declaration)target_link_libraries(strata_ds4 PRIVATE strata_metal_kernels strata_metal_host)diff --git a/metal/include/metal_compat/cublas_v2.h b/metal/include/metal_compat/cublas_v2.hindex 50a966fd3..8df805130 100644--- a/metal/include/metal_compat/cublas_v2.h+++ b/metal/include/metal_compat/cublas_v2.h@@ -21,6 +21,7 @@ enum cublasOperation_t { CUBLAS_OP_N = 0, CUBLAS_OP_T = 1, CUBLAS_OP_C = 2 };enum cublasComputeType_t { CUBLAS_COMPUTE_16F = 64, CUBLAS_COMPUTE_32F = 68, CUBLAS_COMPUTE_32F_FAST_16F = 74 };enum cublasGemmAlgo_t { CUBLAS_GEMM_DEFAULT = -1, CUBLAS_GEMM_DEFAULT_TENSOR_OP = 99 };enum cudaDataType_t { CUDA_R_32F = 0, CUDA_R_16F = 2, CUDA_R_16BF = 14, CUDA_R_8F_E4M3 = 28 };+using cudaDataType = cudaDataType_t; // the engine names it both waysstatic inline cublasStatus_t cublasCreate(cublasHandle_t* h) {*h = (void*) 1; // the Metal runtime is a singleton: the handle carries no statediff --git a/metal/include/metal_compat/cuda_device_builtins.h b/metal/include/metal_compat/cuda_device_builtins.hindex 95c9b9fde..4c9904552 100644--- a/metal/include/metal_compat/cuda_device_builtins.h+++ b/metal/include/metal_compat/cuda_device_builtins.h@@ -1,3 +1,10 @@+#ifndef __grid_constant__+#define __grid_constant__+#endif+#ifndef __device_builtin__+#define __device_builtin__+#endif+// cuda_device_builtins.h - the CUDA device built-ins, declared for the HOST compile only.//// The DeepSeek kernels are compiled twice in this port: as C++ here, to enumerate exactly which API the device code uses,@@ -16,10 +23,6 @@ static strata_dim3 threadIdx{0, 0, 0}, blockIdx{0, 0, 0}, blockDim{1, 1, 1}, gristatic inline void __syncthreads() {}static inline void __syncwarp(unsigned = 0xffffffffu) {}-static inline unsigned __shfl_down_sync(unsigned, unsigned v, int) { return v; }-static inline unsigned __shfl_xor_sync(unsigned, unsigned v, int) { return v; }-static inline float __shfl_down_sync(unsigned, float v, int) { return v; }-static inline float __shfl_xor_sync(unsigned, float v, int) { return v; }static inline unsigned __ballot_sync(unsigned, int) { return 0; }static inline int __popc(unsigned v) { return __builtin_popcount(v); }static inline int __ffs(unsigned v) { return __builtin_ffs((int) v); }@@ -144,6 +147,19 @@ static inline int __float2int_rd(float a) { return (int) __builtin_floorf(a); }static inline int __float2int_ru(float a) { return (int) __builtin_ceilf(a); }template <typename T> static inline T __ldg(const T* p) { return *p; }template <typename T> static inline T __ldcv(const T* p) { return *p; }-static inline unsigned __shfl_up_sync(unsigned, unsigned v, int) { return v; }-static inline float __shfl_up_sync(unsigned, float v, int) { return v; }-static inline unsigned __shfl_sync(unsigned, unsigned v, int) { return v; }++// shuffles: one template each, so int/unsigned/float/short all bind (CUDA has overloads; a template is the same thing here)+template <typename T> static inline T __shfl_xor_sync(unsigned, T v, int) { return v; }+template <typename T> static inline T __shfl_down_sync(unsigned, T v, int) { return v; }+template <typename T> static inline T __shfl_up_sync(unsigned, T v, int) { return v; }+template <typename T> static inline T __shfl_sync(unsigned, T v, int) { return v; }++// the integer multiply-high and the bitwise atomics the sampler uses+static inline unsigned __umulhi(unsigned a, unsigned b) { return (unsigned) (((unsigned long long) a * b) >> 32); }+static inline int __mulhi(int a, int b) { return (int) (((long long) a * b) >> 32); }+static inline unsigned atomicOr(unsigned* p, unsigned v) { const unsigned o = *p; *p = o | v; return o; }+static inline unsigned atomicAnd(unsigned* p, unsigned v) { const unsigned o = *p; *p = o & v; return o; }+static inline unsigned atomicXor(unsigned* p, unsigned v) { const unsigned o = *p; *p = o ^ v; return o; }+static inline unsigned atomicExch(unsigned* p, unsigned v) { const unsigned o = *p; *p = v; return o; }+static inline unsigned atomicInc(unsigned* p, unsigned v) { const unsigned o = *p; *p = (o >= v ? 0 : o + 1); return o; }+static inline unsigned atomicDec(unsigned* p, unsigned v) { const unsigned o = *p; *p = (o == 0 || o > v ? v : o - 1); return o; }diff --git a/metal/include/metal_compat/cuda_fp16.h b/metal/include/metal_compat/cuda_fp16.hindex 9ee80d538..1f88d298e 100644--- a/metal/include/metal_compat/cuda_fp16.h+++ b/metal/include/metal_compat/cuda_fp16.h@@ -85,3 +85,15 @@ static inline float __fadd_rn(float a, float b) { return a + b; }static inline float __fmaf_rn(float a, float b, float c) { return a * b + c; }static inline float __frcp_rn(float a) { return 1.0f / a; }static inline float __fsqrt_rn(float a) { return __builtin_sqrtf(a); }++// the half2 pack/unpack and arithmetic helpers the trellis and vision paths use+static inline __half2 __halves2half2(__half lo, __half hi) { __half2 h; h.x = lo.x; h.y = hi.x; return h; }+static inline __half2 __half2half2(__half v) { __half2 h; h.x = v.x; h.y = v.x; return h; }+static inline __half __low2half(__half2 h) { __half o; o.x = h.x; return o; }+static inline __half __high2half(__half2 h) { __half o; o.x = h.y; return o; }+static inline float __low2float(__half2 h) { return strata_f16_bits_to_f32(h.x); }+static inline float __high2float(__half2 h) { return strata_f16_bits_to_f32(h.y); }+static inline __half2 __hadd2(__half2 a, __half2 b) { return __floats2half2_rn(__half2float(__low2half(a)) + __half2float(__low2half(b)), __half2float(__high2half(a)) + __half2float(__high2half(b))); }+static inline __half2 __hmul2(__half2 a, __half2 b) { return __floats2half2_rn(__half2float(__low2half(a)) * __half2float(__low2half(b)), __half2float(__high2half(a)) * __half2float(__high2half(b))); }+static inline __half2 __hfma2(__half2 a, __half2 b, __half2 c) { return __floats2half2_rn(__half2float(__low2half(a)) * __half2float(__low2half(b)) + __half2float(__low2half(c)), __half2float(__high2half(a)) * __half2float(__high2half(b)) + __half2float(__high2half(c))); }+static inline __half2 __hneg2(__half2 a) { __half2 h; h.x = a.x ^ 0x8000u; h.y = a.y ^ 0x8000u; return h; }diff --git a/metal/include/metal_compat/cuda_launch.h b/metal/include/metal_compat/cuda_launch.hnew file mode 100644index 000000000..256276210--- /dev/null+++ b/metal/include/metal_compat/cuda_launch.h@@ -0,0 +1,23 @@+// cuda_launch.h - the <<<grid, block, shared, stream>>> launch syntax, as a call.+//+// A C++ compiler cannot parse CUDA's launch syntax, and the Metal port replaces every launch with a dispatch anyway (the grid+// becomes a threadgroup count, the block a threadgroup size). strata_ds4_launch() is that call: the kernel first, then the+// launch shape, then the kernel's own arguments - every parameter generic, because the shapes vary (dim3, plain ints, sizes,+// and the stream type), and the point of this header is to make the sources parse, not to type-check the port.+//+// Until a kernel has a Metal implementation the call stops the run naming the kernel, so the engine reports the first one it+// needs and the port proceeds one kernel at a time.+#pragma once++#include <cstdio>+#include <cstdlib>++template <typename K, typename... A>+static inline void strata_ds4_launch(K kernel, A&&...) {+ static_assert(sizeof(K) > 0, "kernel");+ std::fprintf(stderr, "strata metal: a dsv41 kernel launch has no Metal dispatch yet\n");+ std::abort();+}++// the call sites sit inside namespaces whose first argument would win argument-dependent lookup: force the global one+#define strata_ds4_launch(...) ::strata_ds4_launch(__VA_ARGS__)diff --git a/metal/include/metal_compat/cuda_runtime.h b/metal/include/metal_compat/cuda_runtime.hindex fd7cb93c8..aedad00a5 100644--- a/metal/include/metal_compat/cuda_runtime.h+++ b/metal/include/metal_compat/cuda_runtime.h@@ -1,4 +1,5 @@// The CUDA function-space qualifiers and the few launch/kernel spellings the shared sources use. On Metal they say+#include "cuda_launch.h"// nothing: a "kernel" here is an MSL function compiled at run time and dispatched by the host runtime, so the qualifiers// are empty and __forceinline__ is plain inline. Part of the metal_compat layer (M5 Max, 2026-10-23).#ifndef __host__@@ -42,6 +43,7 @@#endif#include "cuda_device_builtins.h"+#include <cmath>using std::fmaxf;@@ -93,6 +95,12 @@ inline constexpr unsigned cudaEventDefault = strata::gpu::kEventDefault;inline constexpr unsigned cudaEventDisableTiming = strata::gpu::kEventDisableTiming;inline constexpr unsigned cudaEventBlockingSync = strata::gpu::kEventBlockingSync;inline constexpr unsigned cudaStreamNonBlocking = strata::gpu::kStreamNonBlocking;+enum cudaStreamCaptureStatus { cudaStreamCaptureStatusNone = 0, cudaStreamCaptureStatusActive = 1, cudaStreamCaptureStatusInvalidated = 2 };+static inline strata::gpu::Error cudaStreamIsCapturing(strata::gpu::Stream, cudaStreamCaptureStatus* st) {+ if (st) *st = cudaStreamCaptureStatusNone;+ return strata::gpu::kSuccess;+}+enum cudaStreamCaptureMode { cudaStreamCaptureModeGlobal = 0, cudaStreamCaptureModeThreadLocal = 1, cudaStreamCaptureModeRelaxed = 2 };// (the compute capability is asked by the 0.1.45 Turing defaults in verify.cpp: no answer here, which reads as "not a Turing card")enum cudaDeviceAttr {@@ -196,3 +204,10 @@ inline cudaError_t cudaGraphExecDestroy(cudaGraphExec_t x) { return strata::gpu:inline strata::gpu::Error cudaMemcpy2D(void*, size_t, const void*, size_t, size_t, size_t, strata::gpu::CopyKind) {return strata::gpu::kErrorUnsupported; // cudaMemcpy2D_unimplemented_marker}++// occupancy query: the Metal runtime decides its own threadgroup limits, so answer "one block" and let the caller size grids+template <typename K, typename... A>+static inline strata::gpu::Error cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* n, K&&, int, A&&...) {+ if (n) *n = 1;+ return strata::gpu::kSuccess;+}diff --git a/metal/include/metal_compat/mma.h b/metal/include/metal_compat/mma.hindex 6a4fe4617..511071f0e 100644--- a/metal/include/metal_compat/mma.h+++ b/metal/include/metal_compat/mma.h@@ -29,10 +29,10 @@ template <typename Use, int M, int N, int K, typename T, typename Layout = void>std::abort();}-template <typename F, typename T> inline void fill_fragment(F&, T) { strata_metal_wmma_missing("fill_fragment"); }-template <typename F, typename P> inline void load_matrix_sync(F&, const P*, unsigned, unsigned = 0) { strata_metal_wmma_missing("load_matrix_sync"); }-template <typename F> inline void mma_sync(F&, const F&, const F&, const F&) { strata_metal_wmma_missing("mma_sync"); }-template <typename F, typename P> inline void store_matrix_sync(P*, const F&, unsigned, layout_t) { strata_metal_wmma_missing("store_matrix_sync"); }+template <typename... A> inline void fill_fragment(A&&...) { strata_metal_wmma_missing("fill_fragment"); }+template <typename... A> inline void load_matrix_sync(A&&...) { strata_metal_wmma_missing("load_matrix_sync"); }+template <typename... A> inline void mma_sync(A&&...) { strata_metal_wmma_missing("mma_sync"); } // variadic: the engine passes four fragments of different types+template <typename... A> inline void store_matrix_sync(A&&...) { strata_metal_wmma_missing("store_matrix_sync"); }} // namespace wmma} // namespace nvcudadiff --git a/metal/src/engine/exl3_legacy_stub.cpp b/metal/src/engine/exl3_legacy_stub.cppnew file mode 100644index 000000000..7d2cd4ce2--- /dev/null+++ b/metal/src/engine/exl3_legacy_stub.cpp@@ -0,0 +1,29 @@+// metal/src/engine/exl3_legacy_stub.cpp - v1.1.8's EXL3 decode kernels, absent on Metal.+//+// exl3_legacy.cu exists for NVIDIA GPUs before sm_80, which pick it over the newer tiles; nothing on a Mac can select it+// (use_legacy() is false without CUDA), and the 2.25bpw text model does not use EXL3 at all. Leaving the file out of the+// target would break the link for these seven entry points, so they are defined here and stop the run naming themselves -+// the same contract as the stub generator, and a loud failure rather than a silent wrong decode.+#include <cstdio>+#include <cstdlib>++#include <strata/dsv41/exl3.hpp> // MvJob, MoeArgs, WinArgs++namespace strata::dsv41k::exl3::legacy {++[[noreturn]] static void missing(const char* what) {+ std::fprintf(stderr,+ "strata metal: exl3::legacy::%s has no Metal implementation (v1.1.8's EXL3 kernels; the 2.25bpw text model "+ "does not use EXL3)\n", what);+ std::abort();+}++int launch_errors() { return 0; }+bool prepare(cudaStream_t) { missing("prepare"); }+bool several_ok() { return false; }+bool mv(const MvJob*, int, cudaStream_t) { missing("mv"); }+void moe_gate_up(const MoeArgs&, const float*, float*, cudaStream_t) { missing("moe_gate_up"); }+void moe_down(const MoeArgs&, const float*, const float*, float*, cudaStream_t) { missing("moe_down"); }+bool win_experts(const WinArgs&, cudaStream_t) { missing("win_experts"); }++} // namespace strata::dsv41k::exl3::legacydiff --git a/src/dsv41/ds4_attn_split.cuh b/src/dsv41/ds4_attn_split.cuhindex 959830b09..9def53b8c 100644--- a/src/dsv41/ds4_attn_split.cuh+++ b/src/dsv41/ds4_attn_split.cuh@@ -198,8 +198,8 @@ void sparse_attn_split(const float* q, const uint16_t* win, const uint8_t* ckv,const AttnRows& ar, const float* sink, int heads, int T, float scale, float* out, const AttnSplitScratch& sc,int total_max, cudaStream_t s) {const int nsplit = (total_max + R - 1) / R;- attn_part<G, R><<<dim3(heads / G, nsplit, T), 256, 0, s>>>(q, win, ckv, ckv_row_bytes, sel, sel_stride, ar, heads, scale, sc.part, sc.ml, sc.nsplit);- attn_merge<<<dim3(heads, T), 256, 0, s>>>(sc.part, sc.ml, sink, heads, sc.nsplit, R, ar, out);+ strata_ds4_launch(attn_part<G, R>, dim3(heads / G, nsplit, T), 256, 0, s, q, win, ckv, ckv_row_bytes, sel, sel_stride, ar, heads, scale, sc.part, sc.ml, sc.nsplit);+ strata_ds4_launch(attn_merge, dim3(heads, T), 256, 0, s, sc.part, sc.ml, sink, heads, sc.nsplit, R, ar, out);}} // namespace attnsplitdiff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 5bee58804..a72bcf7c8 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -198,7 +198,7 @@ __global__ void __launch_bounds__(kThreads) q8_quant_kernel(const float* __restr}void q8_quant(const float* x, size_t ldx, int rows, int k, int8_t* q, float* d, cudaStream_t s) {const dim3 grid((unsigned) ((k / 32 + kWarps - 1) / kWarps), (unsigned) rows);- q8_quant_kernel<<<grid, kThreads, 0, s>>>(x, ldx, k, q, d);+ strata_ds4_launch(q8_quant_kernel, grid, kThreads, 0, s, x, ldx, k, q, d);}// 8 E2M1 codes (a word of 4 code bytes) -> their values as int8: .x the low nibbles' (values 0..3 of the word's bytes),@@ -1040,14 +1040,14 @@ template <int NC>void mv_launch(int f, const BfJobs& J, int nj, int max_n, bool vec, cudaStream_t s) {const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);if (f == 1) {- if (vec) mx_mv_kernel<NC, true, false><<<grid, kThreads, 0, s>>>(J);- else mx_mv_kernel<NC, false, false><<<grid, kThreads, 0, s>>>(J);+ if (vec) strata_ds4_launch(mx_mv_kernel<NC, true, false>, grid, kThreads, 0, s, J);+ else strata_ds4_launch(mx_mv_kernel<NC, false, false>, grid, kThreads, 0, s, J);} else if (f == 2) { // (vec here: every row 4-byte aligned, so staged)- if (vec) q8_mv_kernel<NC, true><<<grid, kThreads, 0, s>>>(J);- else q8_mv_kernel<NC, false><<<grid, kThreads, 0, s>>>(J);+ if (vec) strata_ds4_launch(q8_mv_kernel<NC, true>, grid, kThreads, 0, s, J);+ else strata_ds4_launch(q8_mv_kernel<NC, false>, grid, kThreads, 0, s, J);} else {- if (vec) bf16_mv_kernel<NC, true><<<grid, kThreads, 0, s>>>(J);- else bf16_mv_kernel<NC, false><<<grid, kThreads, 0, s>>>(J);+ if (vec) strata_ds4_launch(bf16_mv_kernel<NC, true>, grid, kThreads, 0, s, J);+ else strata_ds4_launch(bf16_mv_kernel<NC, false>, grid, kThreads, 0, s, J);}}void mv_set(int f, const BfJobs& J, int nj, int max_n, int max_c, bool vec, cudaStream_t s) {@@ -1165,8 +1165,8 @@ void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {const size_t ld = row_stride(m);const size_t total = (size_t) m.n * (size_t) m.k;const unsigned blocks = (unsigned) std::min<size_t>((total + 255) / 256, 65535);- if (m.fmt == kFmtQ8_0) q8_rows_kernel<<<blocks, 256, 0, s>>>((const uint8_t*) m.raw, ld, m.k, m.n, dst, bf16);- else bf16_rows_kernel<<<blocks, 256, 0, s>>>((const uint16_t*) m.raw, ld, m.k, m.n, dst, bf16);+ if (m.fmt == kFmtQ8_0) strata_ds4_launch(q8_rows_kernel, blocks, 256, 0, s, (const uint8_t*) m.raw, ld, m.k, m.n, dst, bf16);+ else strata_ds4_launch(bf16_rows_kernel, blocks, 256, 0, s, (const uint16_t*) m.raw, ld, m.k, m.n, dst, bf16);check("rows");}@@ -1175,15 +1175,15 @@ namespace {template <int NE>struct GateUpK {static void run(const MxArgs& m, dim3 grid, cudaStream_t s) {- if (mx_row_bytes(m.n_embd) % 4 == 0) mx_gate_up_kernel<NE, true><<<grid, kThreads, 0, s>>>(m);- else mx_gate_up_kernel<NE, false><<<grid, kThreads, 0, s>>>(m);+ if (mx_row_bytes(m.n_embd) % 4 == 0) strata_ds4_launch(mx_gate_up_kernel<NE, true>, grid, kThreads, 0, s, m);+ else strata_ds4_launch(mx_gate_up_kernel<NE, false>, grid, kThreads, 0, s, m);}};template <int NE>struct DownK {static void run(const MxArgs& m, dim3 grid, cudaStream_t s) {- if (mx_row_bytes(m.n_ff) % 4 == 0) mx_down_kernel<NE, true><<<grid, kThreads, 0, s>>>(m);- else mx_down_kernel<NE, false><<<grid, kThreads, 0, s>>>(m);+ if (mx_row_bytes(m.n_ff) % 4 == 0) strata_ds4_launch(mx_down_kernel<NE, true>, grid, kThreads, 0, s, m);+ else strata_ds4_launch(mx_down_kernel<NE, false>, grid, kThreads, 0, s, m);}};template <template <int> class Kern>@@ -1265,7 +1265,7 @@ void moe_down(const x3::MoeArgs& a, const float* h, const float* sh_out, float*c.out_stride = (size_t) a.n_embd;c.k = a.k;c.n_embd = a.n_embd;- mx_combine_kernel<<<dim3((unsigned) ((a.n_embd + 127) / 128), 1), 128, 0, s>>>(c);+ strata_ds4_launch(mx_combine_kernel, dim3((unsigned) ((a.n_embd + 127) / 128), 1), 128, 0, s, c);check("moe_down");}@@ -1312,7 +1312,7 @@ bool win_experts(const x3::WinArgs& a, cudaStream_t s) {q8_quant(a.h, (size_t) a.n_ff, a.T * 8, a.n_ff, sc->hq, sc->hd, s); // (rows of picks not on the device: unused)if (mx) launch_groups<DownK>(m, a.n_embd, groups, a.T, s);else iq::down(ia, groups, a.T, s);- if (a.wait_ptr != nullptr) mx_wait_kernel<<<1, 32, 0, s>>>(a.wait_ptr, a.wait_val);+ if (a.wait_ptr != nullptr) strata_ds4_launch(mx_wait_kernel, 1, 32, 0, s, a.wait_ptr, a.wait_val);CombArgs c{};c.plan_ptr = a.plan_ptr;c.plan_w = a.plan_w;@@ -1325,7 +1325,7 @@ bool win_experts(const x3::WinArgs& a, cudaStream_t s) {c.out_stride = a.out_stride;c.k = a.k;c.n_embd = a.n_embd;- mx_combine_kernel<<<dim3((unsigned) ((a.n_embd + 127) / 128), (unsigned) a.T), 128, 0, s>>>(c);+ strata_ds4_launch(mx_combine_kernel, dim3((unsigned) ((a.n_embd + 127) / 128), (unsigned) a.T), 128, 0, s, c);check("win_experts");return true;}@@ -1400,12 +1400,12 @@ void set_gate_up(const x3::SetArgs& sa, const float* x, int ldx, const int* row_k.ldo = ldh;const dim3 grid((unsigned) ((sa.max_rows + BM - 1) / BM), (unsigned) (sa.n_ff / BN), (unsigned) sa.n_exp);#if !defined(STRATA_USE_HIP)- if (tc_ok()) mx_set_tc_kernel<true><<<grid, kThreads, 0, s>>>(k);+ if (tc_ok()) strata_ds4_launch(mx_set_tc_kernel<true>, grid, kThreads, 0, s, k);#else- if (tc_ok()) mx_set_wmma_kernel<true><<<grid, kThreads, 0, s>>>(k);+ if (tc_ok()) strata_ds4_launch(mx_set_wmma_kernel<true>, grid, kThreads, 0, s, k);#endifelse- mx_set_kernel<true><<<grid, kThreads, 0, s>>>(k);+ strata_ds4_launch(mx_set_kernel<true>, grid, kThreads, 0, s, k);check("set_gate_up");}@@ -1457,12 +1457,12 @@ void set_down(const x3::SetArgs& sa, const uint16_t* h16, int ldh, float* y, intk.ldo = ldy;const dim3 grid((unsigned) ((sa.max_rows + BM - 1) / BM), (unsigned) (sa.n_embd / BN), (unsigned) sa.n_exp);#if !defined(STRATA_USE_HIP)- if (tc_ok()) mx_set_tc_kernel<false><<<grid, kThreads, 0, s>>>(k);+ if (tc_ok()) strata_ds4_launch(mx_set_tc_kernel<false>, grid, kThreads, 0, s, k);#else- if (tc_ok()) mx_set_wmma_kernel<false><<<grid, kThreads, 0, s>>>(k);+ if (tc_ok()) strata_ds4_launch(mx_set_wmma_kernel<false>, grid, kThreads, 0, s, k);#endifelse- mx_set_kernel<false><<<grid, kThreads, 0, s>>>(k);+ strata_ds4_launch(mx_set_kernel<false>, grid, kThreads, 0, s, k);check("set_down");}diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex e039c4d66..1baa84e5d 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -952,9 +952,9 @@ bool tc_ok() { // as ds4_gguf.cu's: sm_70 and later, STRATA_DS4_NO_TC=1 keeps}template <int T, int NE>-void launch_gu(const Args& a, dim3 grid, cudaStream_t s) { gate_up_kernel<T, NE><<<grid, kThreads, 0, s>>>(a); }+void launch_gu(const Args& a, dim3 grid, cudaStream_t s) { strata_ds4_launch(gate_up_kernel<T, NE>, grid, kThreads, 0, s, a); }template <int T, int NE>-void launch_dn(const Args& a, dim3 grid, cudaStream_t s) { down_kernel<T, NE><<<grid, kThreads, 0, s>>>(a); }+void launch_dn(const Args& a, dim3 grid, cudaStream_t s) { strata_ds4_launch(down_kernel<T, NE>, grid, kThreads, 0, s, a); }template <int T>void gu_ne(const Args& a, dim3 grid, int ne, cudaStream_t s) {if (ne <= 1) launch_gu<T, 1>(a, grid, s);@@ -973,16 +973,16 @@ template <int T, bool GU>void launch_set(const SetArgs& a, dim3 grid, cudaStream_t s) {#if !defined(STRATA_USE_HIP)if (tc_ok()) {- set_tc_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);+ strata_ds4_launch(set_tc_kernel<T, GU>, grid, kThreads, 0, s, a);return;}#endifif (a.max_rows <= 16) { // (a block of 64 rows would do 64 rows' FMAs for the few there are)grid.x = (unsigned) ((a.max_rows + 15) / 16);- set_kernel<T, GU, 16><<<grid, kThreads, 0, s>>>(a);+ strata_ds4_launch(set_kernel<T, GU, 16>, grid, kThreads, 0, s, a);return;}- set_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);+ strata_ds4_launch(set_kernel<T, GU>, grid, kThreads, 0, s, a);}// the light sets' int8 rows, per stream (kLightMaxExp kLightRows rows of k)struct LightBuf {@@ -1018,16 +1018,16 @@ bool light_on() { // STRATA_DS4_NO_LIGHT=1: every set through the GEMM tilestemplate <int T, bool GU>void launch_light(const SetArgs& a, const LightBuf& b, cudaStream_t s) {const dim3 grid((unsigned) (((GU ? a.n_ff : a.n_embd) + kWarps - 1) / kWarps), (unsigned) a.n_exp);- if (a.max_rows <= 4) light_kernel<T, 4, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);- else if (a.max_rows <= 8) light_kernel<T, 8, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);- else light_kernel<T, kLightRows, GU><<<grid, kThreads, 0, s>>>(a, b.q, b.d);+ if (a.max_rows <= 4) strata_ds4_launch(light_kernel<T, 4, GU>, grid, kThreads, 0, s, a, b.q, b.d);+ else if (a.max_rows <= 8) strata_ds4_launch(light_kernel<T, 8, GU>, grid, kThreads, 0, s, a, b.q, b.d);+ else strata_ds4_launch(light_kernel<T, kLightRows, GU>, grid, kThreads, 0, s, a, b.q, b.d);}bool light_set(const SetArgs& a, bool gu, cudaStream_t s) {const int K = gu ? a.n_embd : a.n_ff;LightBuf* b = light_buf(s, K);if (b == nullptr) return false;const dim3 gq((unsigned) ((K / 32 + kWarps - 1) / kWarps), (unsigned) (a.n_exp * kLightRows));- light_q8_kernel<<<gq, kThreads, 0, s>>>(gu ? a.xf : nullptr, gu ? nullptr : a.xh, a.idx, a.ldx, K, a.bounds,+ strata_ds4_launch(light_q8_kernel, gq, kThreads, 0, s, gu ? a.xf : nullptr, gu ? nullptr : a.xh, a.idx, a.ldx, K, a.bounds,a.row0, b->q, b->d);if (gu) {if (a.tg == kMXFP4) {@@ -1052,9 +1052,9 @@ bool light_set(const SetArgs& a, bool gu, cudaStream_t s) {template <int T>void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {const dim3 grid((unsigned) ((max_n + kWarps - 1) / kWarps), (unsigned) nj);- if (max_c <= 1) mv_kernel<T, 1><<<grid, kThreads, 0, s>>>(J);- else if (max_c <= 4) mv_kernel<T, 4><<<grid, kThreads, 0, s>>>(J);- else mv_kernel<T, 16><<<grid, kThreads, 0, s>>>(J);+ if (max_c <= 1) strata_ds4_launch(mv_kernel<T, 1>, grid, kThreads, 0, s, J);+ else if (max_c <= 4) strata_ds4_launch(mv_kernel<T, 4>, grid, kThreads, 0, s, J);+ else strata_ds4_launch(mv_kernel<T, 16>, grid, kThreads, 0, s, J);}@@ -1091,7 +1091,7 @@ __global__ void __launch_bounds__(kThreads) dq_rows_kernel(const uint8_t* __restbool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {if (raw == nullptr || k % 256 != 0 || n <= 0) return false;const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);-#define DS4_DQ(T) dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16)+#define DS4_DQ(T) strata_ds4_launch(dq_rows_kernel<T>, blocks, kThreads, 0, s, (const uint8_t*) raw, ld, k, n, dst, bf16)switch (type) {case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;case kIQ2_XS: DS4_DQ(kIQ2_XS); break;diff --git a/src/dsv41/ds4_kernels.cu b/src/dsv41/ds4_kernels.cuindex b04e9eec9..956ccb501 100644--- a/src/dsv41/ds4_kernels.cu+++ b/src/dsv41/ds4_kernels.cu@@ -1823,15 +1823,15 @@ void route_publish(const float* logits, const float* bias, int n, int k, int T,const unsigned long long* vram, const float* x, int dim, int* idx, float* w,unsigned long long* plan_ptr, Req* req, unsigned seq, cudaStream_t s, const float* bias_vl,unsigned vl, int na0, int na1) {- route_publish_kernel<<<T, 512, 0, s>>>(logits, bias, n, k, route_scale, vram, x, dim, idx, w, plan_ptr, req,+ strata_ds4_launch(route_publish_kernel, T, 512, 0, s, logits, bias, n, k, route_scale, vram, x, dim, idx, w, plan_ptr, req,vl != 0 ? bias_vl : nullptr, vl);- if (req != nullptr) publish_kernel<<<1, 1, 0, s>>>(req, T, seq, na0, na1);+ if (req != nullptr) strata_ds4_launch(publish_kernel, 1, 1, 0, s, req, T, seq, na0, na1);}void route_ahead(const float* logits, const float* bias, int n, int m, int T, Req* req, int off, cudaStream_t s) {if (req != nullptr && m > 0 && T > 0 && n <= 1024 && off + m <= kAhead)- route_ahead_kernel<<<T, 512, 0, s>>>(logits, bias, n, m, req, off);+ strata_ds4_launch(route_ahead_kernel, T, 512, 0, s, logits, bias, n, m, req, off);}-void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) { wait_seq_kernel<<<1, 32, 0, s>>>(seq_ptr, seq); }+void wait_seq(const unsigned* seq_ptr, unsigned seq, cudaStream_t s) { strata_ds4_launch(wait_seq_kernel, 1, 32, 0, s, seq_ptr, seq); }// the RAM store's experts for the device (a decode token, window of one): a pick the VRAM table left at 0 whose expert the store holds// (zc_tab[expert] = the slot's device address over PCIe, 0: not) is copied to staging slot i with wide coalesced loads (the PCIe@@ -1855,14 +1855,14 @@ __global__ void zc_fix_kernel(unsigned long long* __restrict__ plan, const int*}void zc_stage(unsigned long long* plan, const int* idx, const unsigned long long* zc_tab, int k, uint8_t* stage, size_t stride, size_t bytes,cudaStream_t s) {- zc_copy_kernel<<<dim3(64, k), 256, 0, s>>>(plan, idx, zc_tab, stage, stride, bytes);- zc_fix_kernel<<<1, 32, 0, s>>>(plan, idx, zc_tab, k, stage, stride);+ strata_ds4_launch(zc_copy_kernel, dim3(64, k), 256, 0, s, plan, idx, zc_tab, stage, stride, bytes);+ strata_ds4_launch(zc_fix_kernel, 1, 32, 0, s, plan, idx, zc_tab, k, stage, stride);}-void init_pre(float* pre, int T, cudaStream_t s) { init_pre_kernel<<<cdiv(T * 8, 128), 128, 0, s>>>(pre, T); }+void init_pre(float* pre, int T, cudaStream_t s) { strata_ds4_launch(init_pre_kernel, cdiv(T * 8, 128), 128, 0, s, pre, T); }void embed(const uint16_t* rows, int T, float* h, int dim, int hc, cudaStream_t s) {- embed_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(rows, h, dim, hc);+ strata_ds4_launch(embed_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, rows, h, dim, hc);}void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudaStream_t s) {@@ -1876,76 +1876,76 @@ void finish(float* x, int n_vec, int len, long long stride, const Finish& f, cudstd::fprintf(stderr, "ds4 finish: packed rows of quant %d, %d values not covered\n", f.quant, len);return;}- finish_kernel<true><<<n_vec, kThreads, 0, s>>>(x, len, stride, f);+ strata_ds4_launch(finish_kernel<true>, n_vec, kThreads, 0, s, x, len, stride, f);} else- finish_kernel<false><<<n_vec, kThreads, 0, s>>>(x, len, stride, f);+ strata_ds4_launch(finish_kernel<false>, n_vec, kThreads, 0, s, x, len, stride, f);}void hc_mix(const float* h, const float* fn, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {- hc_mix_kernel<<<dim3(n_mix, T), kThreads, 0, s>>>(h, fn, hcdim, eps, mixes);+ strata_ds4_launch(hc_mix_kernel, dim3(n_mix, T), kThreads, 0, s, h, fn, hcdim, eps, mixes);}void hc_split(const float* mixes, const float* scale, const float* base, int hc, int iters, float hc_eps, int T,float* pre, float* post, float* comb, cudaStream_t s) {- hc_split_kernel<<<T, 32, 0, s>>>(mixes, scale, base, hc, iters, hc_eps, pre, post, comb);+ strata_ds4_launch(hc_split_kernel, T, 32, 0, s, mixes, scale, base, hc, iters, hc_eps, pre, post, comb);}void hc_pre_norm(const float* h, const float* mix, const float* w, int dim, int hc, int T, float eps, float* x,cudaStream_t s) {- hc_pre_norm_kernel<<<T, 1024, 0, s>>>(h, mix, w, dim, hc, eps, x);+ strata_ds4_launch(hc_pre_norm_kernel, T, 1024, 0, s, h, mix, w, dim, hc, eps, x);}void hc_post(const float* y, const float* res, const float* post, const float* comb, int dim, int hc, int T,float* out, cudaStream_t s) {- hc_post_kernel<<<dim3(cdiv(dim * hc, kThreads), T), kThreads, 0, s>>>(y, res, post, comb, dim, hc, out);+ strata_ds4_launch(hc_post_kernel, dim3(cdiv(dim * hc, kThreads), T), kThreads, 0, s, y, res, post, comb, dim, hc, out);}void hc_post_inplace(const float* y, float* h, const float* post, const float* comb, int dim, int hc, int T,cudaStream_t s) {- hc_post_inplace_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(y, h, post, comb, dim, hc);+ strata_ds4_launch(hc_post_inplace_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, y, h, post, comb, dim, hc);}void hc_mix_scale(const float* h, int hcdim, int n_mix, int T, float eps, float* mixes, cudaStream_t s) {- hc_mix_scale_kernel<<<T, kThreads, 0, s>>>(h, hcdim, n_mix, eps, mixes);+ strata_ds4_launch(hc_mix_scale_kernel, T, kThreads, 0, s, h, hcdim, n_mix, eps, mixes);}void engram_dequant_tok(const uint8_t* src, size_t per, int cols, int T, int hd, float* out, cudaStream_t s, int fmt) {- if (T > 0) engram_dequant_tok_kernel<<<T * cols, kThreads, 0, s>>>(src, per, cols, hd, out, fmt);+ if (T > 0) strata_ds4_launch(engram_dequant_tok_kernel, T * cols, kThreads, 0, s, src, per, cols, hd, out, fmt);}void moe_sort(const int* idx, int T, int k, const int* vs, int n_vs, int* bounds, int* row_tok, int* pos,cudaStream_t s) {- moe_sort_kernel<false><<<n_vs, kThreads, 0, s>>>(idx, T, k, vs, bounds, row_tok, pos);- bounds_scan_kernel<<<1, 32, 0, s>>>(bounds, n_vs);- moe_sort_kernel<true><<<n_vs, kThreads, 0, s>>>(idx, T, k, vs, bounds, row_tok, pos);+ strata_ds4_launch(moe_sort_kernel<false>, n_vs, kThreads, 0, s, idx, T, k, vs, bounds, row_tok, pos);+ strata_ds4_launch(bounds_scan_kernel, 1, 32, 0, s, bounds, n_vs);+ strata_ds4_launch(moe_sort_kernel<true>, n_vs, kThreads, 0, s, idx, T, k, vs, bounds, row_tok, pos);}void moe_combine(float* acc, const float* y, const int* pos, const float* w, int T, int k, int r0, int r1, int dim,cudaStream_t s) {- if (r1 > r0 && T > 0) moe_combine_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(acc, y, pos, w, k, r0, r1, dim);+ if (r1 > r0 && T > 0) strata_ds4_launch(moe_combine_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, acc, y, pos, w, k, r0, r1, dim);}void gemv_f16(const uint16_t* W, const float* x, int n, int k, int T, float alpha, float* y, cudaStream_t s) {- gemv_f16_kernel<<<dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s>>>(W, x, n, k, alpha, y);+ strata_ds4_launch(gemv_f16_kernel, dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s, W, x, n, k, alpha, y);}void gemv_f16_rows(const uint16_t* W, const float* x, size_t ldx, int n, int k, int T, float alpha, float* y, size_t ldy,cudaStream_t s) {if (T > 0)- gemv_f16_rows_kernel<<<dim3(cdiv(n, kThreads / 32), cdiv(T, kRowsT)), kThreads, 0, s>>>(W, x, ldx, n, k, T, alpha,+ strata_ds4_launch(gemv_f16_rows_kernel, dim3(cdiv(n, kThreads / 32), cdiv(T, kRowsT)), kThreads, 0, s, W, x, ldx, n, k, T, alpha,y, ldy);}void gemv_f32(const float* W, const float* x, int n, int k, int T, float* y, int ldy, cudaStream_t s) {- if (T > 0) gemv_f32_kernel<<<dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s>>>(W, x, n, k, y, ldy);+ if (T > 0) strata_ds4_launch(gemv_f32_kernel, dim3(cdiv(n, kThreads / 32), T), kThreads, 0, s, W, x, n, k, y, ldy);}void route(const float* logits, const float* bias, int n, int k, int T, float route_scale, int* idx, float* w,cudaStream_t s, const float* bias_vl, const VlBits* vl) {static const VlBits none = {};- route_kernel<<<T, 512, 0, s>>>(logits, bias, n, k, route_scale, idx, w, vl != nullptr ? bias_vl : nullptr,+ strata_ds4_launch(route_kernel, T, 512, 0, s, logits, bias, n, k, route_scale, idx, w, vl != nullptr ? bias_vl : nullptr,vl != nullptr ? *vl : none);}void swiglu(const float* g, const float* u, int n, float lim, float* h, cudaStream_t s) {- if (n > 0) swiglu_kernel<<<cdiv(n, kThreads), kThreads, 0, s>>>(g, u, n, lim, h);+ if (n > 0) strata_ds4_launch(swiglu_kernel, cdiv(n, kThreads), kThreads, 0, s, g, u, n, lim, h);}void compress_win(const float* kv, const float* score, float* kv_state, float* score_state, int ratio, int p0, int T,int d, const float* w, float eps, float* latent, cudaStream_t s) {- compress_kernel<<<1, 512, 0, s>>>(kv, score, kv_state, score_state, ratio, p0, T, d, w, eps, latent);+ strata_ds4_launch(compress_kernel, 1, 512, 0, s, kv, score, kv_state, score_state, ratio, p0, T, d, w, eps, latent);}void index_score(const float* q, const float* w, const void* k, int heads, int hd, int T, int n, const int* cand,@@ -1959,16 +1959,16 @@ void index_score(const float* q, const float* w, const void* k, int heads, int hconst size_t smem = (size_t) (heads * hd + heads) * sizeof(float);const dim3 grid(cdiv(n, kThreads), T);if (kfmt == kKvFp4p)- index_score_kernel<128, kKvFp4p><<<grid, kThreads, smem, s>>>(q, w, (const uint8_t*) k, heads, n, cand,+ strata_ds4_launch(index_score_kernel<128, kKvFp4p>, grid, kThreads, smem, s, q, w, (const uint8_t*) k, heads, n, cand,cand_stride, blk, lim, out, out_stride);else- index_score_kernel<128, kKvBf16><<<grid, kThreads, smem, s>>>(q, w, (const uint8_t*) k, heads, n, cand,+ strata_ds4_launch(index_score_kernel<128, kKvBf16>, grid, kThreads, smem, s, q, w, (const uint8_t*) k, heads, n, cand,cand_stride, blk, lim, out, out_stride);}void topk(const float* vals, long long stride, int T, const Rows& r, const int* cand, int cand_stride, int blk,int* out, int out_stride, cudaStream_t s) {- topk_kernel<<<T, 1024, 0, s>>>(vals, stride, r, cand, cand_stride, blk, out, out_stride);+ strata_ds4_launch(topk_kernel, T, 1024, 0, s, vals, stride, r, cand, cand_stride, blk, out, out_stride);}void cand_blocks(const float* score, long long stride, int T, const Rows& r, int blk, float* bm, long long bm_stride,@@ -1985,7 +1985,7 @@ void cand_blocks(const float* score, long long stride, int T, const Rows& r, intrb.k[t] = r.k[t];nb_max = rb.n[t] > nb_max ? rb.n[t] : nb_max;}- block_max_kernel<<<dim3(cdiv(nb_max, kThreads), T), kThreads, 0, s>>>(score, stride, r, blk, bm, bm_stride);+ strata_ds4_launch(block_max_kernel, dim3(cdiv(nb_max, kThreads), T), kThreads, 0, s, score, stride, r, blk, bm, bm_stride);topk(bm, bm_stride, T, rb, nullptr, 0, 1, out, out_stride, s);}@@ -2010,7 +2010,7 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const intreturn;}if (ckv_fmt == kKvBf16) {- sparse_attn_kernel<false><<<dim3(heads, T), kThreads, 0, s>>>(q, win, (const uint16_t*) ckv, sel, sel_stride, a,+ strata_ds4_launch(sparse_attn_kernel<false>, dim3(heads, T), kThreads, 0, s, q, win, (const uint16_t*) ckv, sel, sel_stride, a,sink, heads, d, scale, out);return;}@@ -2033,15 +2033,15 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const int// the BF16 rows' kernel itself (heads_per_block -1: this way at any T)const bool few = heads_per_block == -1 || (heads_per_block == 0 && T <= kAttnScratchT);if (few && scr != nullptr && scr->rows != nullptr && T <= scr->max_t && sel_stride <= scr->cap) {- unpack_sel_kernel<<<dim3(cdiv(sel_stride, kStage), T), kThreads, 0, s>>>((const uint8_t*) ckv, sel, sel_stride, a,+ strata_ds4_launch(unpack_sel_kernel, dim3(cdiv(sel_stride, kStage), T), kThreads, 0, s, (const uint8_t*) ckv, sel, sel_stride, a,scr->cap, scr->rows);#if !defined(STRATA_USE_HIP)if (T == 1) // (V100: the loads ahead pay for the decode's kernel at one query, not at more)- sparse_attn_kernel<true, 8><<<dim3(heads, T), kThreads, 0, s>>>(q, win, scr->rows, nullptr, scr->cap, a, sink,+ strata_ds4_launch(sparse_attn_kernel<true, 8>, dim3(heads, T), kThreads, 0, s, q, win, scr->rows, nullptr, scr->cap, a, sink,heads, d, scale, out);else#endif- sparse_attn_kernel<true, 0><<<dim3(heads, T), kThreads, 0, s>>>(q, win, scr->rows, nullptr, scr->cap, a, sink,+ strata_ds4_launch(sparse_attn_kernel<true, 0>, dim3(heads, T), kThreads, 0, s, q, win, scr->rows, nullptr, scr->cap, a, sink,heads, d, scale, out);return;}@@ -2054,86 +2054,86 @@ void sparse_attn(const float* q, const uint16_t* win, const void* ckv, const intif (heads % G != 0) G = 1;const dim3 grid(heads / G, T);const uint8_t* c = (const uint8_t*) ckv;- if (G == 8) sparse_attn_packed_kernel<8><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);- else if (G == 4) sparse_attn_packed_kernel<4><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);- else if (G == 2) sparse_attn_packed_kernel<2><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);- else sparse_attn_packed_kernel<1><<<grid, kThreads, 0, s>>>(q, win, c, sel, sel_stride, a, sink, heads, scale, out);+ if (G == 8) strata_ds4_launch(sparse_attn_packed_kernel<8>, grid, kThreads, 0, s, q, win, c, sel, sel_stride, a, sink, heads, scale, out);+ else if (G == 4) strata_ds4_launch(sparse_attn_packed_kernel<4>, grid, kThreads, 0, s, q, win, c, sel, sel_stride, a, sink, heads, scale, out);+ else if (G == 2) strata_ds4_launch(sparse_attn_packed_kernel<2>, grid, kThreads, 0, s, q, win, c, sel, sel_stride, a, sink, heads, scale, out);+ else strata_ds4_launch(sparse_attn_packed_kernel<1>, grid, kThreads, 0, s, q, win, c, sel, sel_stride, a, sink, heads, scale, out);}void ring_to_ext(const uint8_t* ring, uint16_t* ext, int64_t p0, int n, int d, cudaStream_t s) {- ring_to_ext_kernel<<<n, kThreads, 0, s>>>(ring, ext, (long long) p0, n, d);+ strata_ds4_launch(ring_to_ext_kernel, n, kThreads, 0, s, ring, ext, (long long) p0, n, d);}void ext_to_ring(const uint16_t* ext, uint8_t* ring, int64_t p0, int n, int T, int d, cudaStream_t s, uint8_t* backup) {if (d % 32 != 0) {std::fprintf(stderr, "ds4 ext_to_ring: row of %d not covered\n", d);return;}- ext_to_ring_kernel<<<T, kThreads, 0, s>>>(ext, ring, (long long) p0, n, T, d, backup);+ strata_ds4_launch(ext_to_ring_kernel, T, kThreads, 0, s, ext, ring, (long long) p0, n, T, d, backup);}void ring_restore(uint8_t* ring, const uint8_t* backup, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {if (T > keep)- ring_restore_kernel<<<T - keep, kThreads, 0, s>>>(ring, backup, (long long) p0, n, keep, kv_row_bytes(kKvFp8, d));+ strata_ds4_launch(ring_restore_kernel, T - keep, kThreads, 0, s, ring, backup, (long long) p0, n, keep, kv_row_bytes(kKvFp8, d));}void ring_restore_batch(const RingBatch& b, int64_t p0, int n, int keep, int T, int d, cudaStream_t s) {if (T > keep && b.n > 0)- ring_restore_batch_kernel<<<dim3(T - keep, b.n), kThreads, 0, s>>>(b, (long long) p0, n, keep,+ strata_ds4_launch(ring_restore_batch_kernel, dim3(T - keep, b.n), kThreads, 0, s, b, (long long) p0, n, keep,kv_row_bytes(kKvFp8, d));}void hc_mean(const float* h, int T, int hc, int dim, float* out, long long out_stride, cudaStream_t s) {- hc_mean_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(h, hc, dim, out, out_stride);+ strata_ds4_launch(hc_mean_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, h, hc, dim, out, out_stride);}-void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { iota_rows_kernel<<<T, 64, 0, s>>>(out, n, stride); }+void iota_rows(int* out, int T, int n, int stride, cudaStream_t s) { strata_ds4_launch(iota_rows_kernel, T, 64, 0, s, out, n, stride); }void markov_add(float* logits, const uint16_t* E, const uint16_t* H, const int* tok, int V, int R, cudaStream_t s) {- markov_kernel<<<cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, E, H, tok, V, R);+ strata_ds4_launch(markov_kernel, cdiv(V, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, E, H, tok, V, R);}void markov_share(float* logits, const uint16_t* H, const uint16_t* erow, int vn, int R, cudaStream_t s) {- markov_share_kernel<<<cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s>>>(logits, H, erow, vn, R);+ strata_ds4_launch(markov_share_kernel, cdiv(vn, kThreads / 32), kThreads, (size_t) R * sizeof(float), s, logits, H, erow, vn, R);}void argmax_share(const float* row, int vn, int v0, unsigned* slot, unsigned seq, cudaStream_t s) {- argmax_share_kernel<<<1, 1024, 0, s>>>(row, vn, v0, slot, seq);+ strata_ds4_launch(argmax_share_kernel, 1, 1024, 0, s, row, vn, v0, slot, seq);}void markov_pick(const unsigned* slots, int nm, unsigned seq, int* tok, const uint16_t* E, uint16_t* erow, int R,cudaStream_t s) {- markov_pick_kernel<<<1, 256, 0, s>>>(slots, nm, seq, tok, E, erow, R);+ strata_ds4_launch(markov_pick_kernel, 1, 256, 0, s, slots, nm, seq, tok, E, erow, R);}void confidence(const float* x, const uint16_t* E, const int* tok, const float* w, int T, int dim, int R, float* out,cudaStream_t s) {- confidence_kernel<<<T, 1024, 0, s>>>(x, E, tok, w, dim, R, out);+ strata_ds4_launch(confidence_kernel, T, 1024, 0, s, x, E, tok, w, dim, R, out);}void engram_dequant(const uint8_t* rows, const uint8_t* scales, int n, int hd, float* out, cudaStream_t s, int fmt) {- engram_dequant_kernel<<<n, kThreads, 0, s>>>(rows, scales, hd, out, fmt);+ strata_ds4_launch(engram_dequant_kernel, n, kThreads, 0, s, rows, scales, hd, out, fmt);}void engram_gate(float* h, const float* kv, const float* w, int hc, int dim, int T, float eps, cudaStream_t s) {- engram_gate_kernel<<<dim3(hc, T), 1024, 0, s>>>(h, kv, w, hc, dim, eps);+ strata_ds4_launch(engram_gate_kernel, dim3(hc, T), 1024, 0, s, h, kv, w, hc, dim, eps);}void gather_rows(const float* x, const Gather& g, int n, int cols, float* out, cudaStream_t s) {- if (n > 0) gather_kernel<<<dim3(cdiv(cols, kThreads), n), kThreads, 0, s>>>(x, g, cols, out);+ if (n > 0) strata_ds4_launch(gather_kernel, dim3(cdiv(cols, kThreads), n), kThreads, 0, s, x, g, cols, out);}void combine(const float* ed, const float* cpu, const float* sh, const Combine& c, int T, int dim, float* out,cudaStream_t s) {- combine_kernel<<<dim3(cdiv(dim, kThreads), T), kThreads, 0, s>>>(ed, cpu, sh, c, dim, out);+ strata_ds4_launch(combine_kernel, dim3(cdiv(dim, kThreads), T), kThreads, 0, s, ed, cpu, sh, c, dim, out);}void xfer_put(const Xfer& x, unsigned* flag, unsigned seq, unsigned* cnt, cudaStream_t s) {size_t total = 0;for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));- xfer_put_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq, cnt);+ strata_ds4_launch(xfer_put_kernel, blocks, kThreads, 0, s, x, flag, seq, cnt);}void xfer_get(const Xfer& x, const unsigned* flag, unsigned seq, cudaStream_t s) {size_t total = 0;for (int i = 0; i < x.n; ++i) total += x.bytes[i] / 16;const int blocks = (int) std::max<size_t>(1, std::min<size_t>(64, (total + kThreads - 1) / kThreads));- xfer_get_kernel<<<blocks, kThreads, 0, s>>>(x, flag, seq);+ strata_ds4_launch(xfer_get_kernel, blocks, kThreads, 0, s, x, flag, seq);}void wait_flags(const unsigned* flags, int n, int stride, unsigned seq, cudaStream_t s) {- if (n > 0) wait_flags_kernel<<<1, 32, 0, s>>>(flags, n, stride, seq);+ if (n > 0) strata_ds4_launch(wait_flags_kernel, 1, 32, 0, s, flags, n, stride, seq);}-void set_flag(unsigned* flag, unsigned v, cudaStream_t s) { set_flag_kernel<<<1, 1, 0, s>>>(flag, v); }+void set_flag(unsigned* flag, unsigned v, cudaStream_t s) { strata_ds4_launch(set_flag_kernel, 1, 1, 0, s, flag, v); }void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsigned* cnt, cudaStream_t s) {const int slot = (int) (seq % (unsigned) a.slots);if (a.nm > 2 && a.sums != nullptr && a.bflags != nullptr) { // (the reduce-scatter: ar_rs_kernel)@@ -2144,9 +2144,9 @@ void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsignedconst int n4 = n / 4;const int blocks = std::max(1, std::min(kArBlocks, cdiv(n4, kThreads)));if (f16)- ar_rs16_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.sums, a.stride, a.bflags, a.nm, mi, slot, a.slots, seq);+ strata_ds4_launch(ar_rs16_kernel, blocks, kThreads, 0, s, x, n4, a.data, a.sums, a.stride, a.bflags, a.nm, mi, slot, a.slots, seq);else- ar_rs_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.sums, a.stride, a.bflags, a.nm, mi, slot, a.slots, seq);+ strata_ds4_launch(ar_rs_kernel, blocks, kThreads, 0, s, x, n4, a.data, a.sums, a.stride, a.bflags, a.nm, mi, slot, a.slots, seq);return;}Xfer xp;@@ -2157,7 +2157,7 @@ void allreduce(float* x, int n, const ArGroup& a, int mi, unsigned seq, unsignedxfer_put(xp, a.flags + ((size_t) mi * a.slots + slot) * 32, seq, cnt, s);const int n4 = n / 4;const int blocks = std::max(1, std::min(64, cdiv(n4, kThreads)));- ar_sum_kernel<<<blocks, kThreads, 0, s>>>(x, n4, a.data, a.stride, a.flags, a.nm, mi, slot, a.slots, seq);+ strata_ds4_launch(ar_sum_kernel, blocks, kThreads, 0, s, x, n4, a.data, a.stride, a.flags, a.nm, mi, slot, a.slots, seq);}} // namespace strata::ds4diff --git a/src/dsv41/ds4_tq.cu b/src/dsv41/ds4_tq.cuindex 724072e67..10c7261f8 100644--- a/src/dsv41/ds4_tq.cu+++ b/src/dsv41/ds4_tq.cu@@ -486,7 +486,7 @@ bool gate_up(const Args& a, int groups, int ne_max, cudaStream_t s) {(void) ne_max;if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("gate_up: geometry not covered");const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) groups);-#define GU_(K, C) { if (w) gate_up_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a); else gate_up_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a); }+#define GU_(K, C) { if (w) strata_ds4_launch(gate_up_kernel<K, C, true>, grid, kThreads, 0, s, a); else strata_ds4_launch(gate_up_kernel<K, C, false>, grid, kThreads, 0, s, a); }DS4_TQ_SWITCH(a.tg, GU_)#undef GU_return true;@@ -497,7 +497,7 @@ bool down(const Args& a, int groups, int ne_max, cudaStream_t s) {(void) ne_max;if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("down: geometry not covered");const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) groups);-#define DN_(K, C) { if (w) down_kernel<K, C, true><<<grid, kThreads, 0, s>>>(a); else down_kernel<K, C, false><<<grid, kThreads, 0, s>>>(a); }+#define DN_(K, C) { if (w) strata_ds4_launch(down_kernel<K, C, true>, grid, kThreads, 0, s, a); else strata_ds4_launch(down_kernel<K, C, false>, grid, kThreads, 0, s, a); }DS4_TQ_SWITCH(a.td, DN_)#undef DN_return true;@@ -507,7 +507,7 @@ bool set_gate_up(const SetArgs& a, cudaStream_t s) {const bool w = use_words();if (a.n_embd % 128 != 0 || a.n_ff % (kWaves * kTileRows) != 0) return bad("set_gate_up: geometry not covered");const dim3 grid((unsigned) (a.n_ff / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);-#define SG_(K, C) { if (w) set_kernel<K, C, true, true><<<grid, kThreads, 0, s>>>(a); else set_kernel<K, C, true, false><<<grid, kThreads, 0, s>>>(a); }+#define SG_(K, C) { if (w) strata_ds4_launch(set_kernel<K, C, true, true>, grid, kThreads, 0, s, a); else strata_ds4_launch(set_kernel<K, C, true, false>, grid, kThreads, 0, s, a); }DS4_TQ_SWITCH(a.tg, SG_)#undef SG_return true;@@ -517,7 +517,7 @@ bool set_down(const SetArgs& a, cudaStream_t s) {const bool w = use_words();if (a.n_ff % 128 != 0 || a.n_embd % (kWaves * kTileRows) != 0) return bad("set_down: geometry not covered");const dim3 grid((unsigned) (a.n_embd / (kWaves * kTileRows)), (unsigned) ((a.max_rows + kMaxEnt - 1) / kMaxEnt), (unsigned) a.n_exp);-#define SD_(K, C) { if (w) set_kernel<K, C, false, true><<<grid, kThreads, 0, s>>>(a); else set_kernel<K, C, false, false><<<grid, kThreads, 0, s>>>(a); }+#define SD_(K, C) { if (w) strata_ds4_launch(set_kernel<K, C, false, true>, grid, kThreads, 0, s, a); else strata_ds4_launch(set_kernel<K, C, false, false>, grid, kThreads, 0, s, a); }DS4_TQ_SWITCH(a.td, SD_)#undef SD_return true;@@ -529,7 +529,7 @@ bool mv(const MvJob* jobs, int n, cudaStream_t s) {const bool w = use_words();if (j.k % 128 != 0 || j.n % kTileRows != 0 || j.ncol < 1 || j.ncol > kMaxEnt) return bad("mv: geometry not covered");const dim3 grid((unsigned) ((j.n + kWaves * kTileRows - 1) / (kWaves * kTileRows)));-#define MV_(K, C) { if (w) mv_kernel<K, C, true><<<grid, kThreads, 0, s>>>(j); else mv_kernel<K, C, false><<<grid, kThreads, 0, s>>>(j); }+#define MV_(K, C) { if (w) strata_ds4_launch(mv_kernel<K, C, true>, grid, kThreads, 0, s, j); else strata_ds4_launch(mv_kernel<K, C, false>, grid, kThreads, 0, s, j); }DS4_TQ_SWITCH(j.type, MV_)#undef MV_}diff --git a/src/dsv41/ds4_vision.cu b/src/dsv41/ds4_vision.cuindex 38916c755..1d94ebe4b 100644--- a/src/dsv41/ds4_vision.cu+++ b/src/dsv41/ds4_vision.cu@@ -446,9 +446,9 @@ bool VisionTower::encode(const uint8_t* rgb, int width, int height, std::vector<// the patchesVK(cudaMemcpyAsync(rgb_, rgb, (size_t) N * 3 * P * P, cudaMemcpyHostToDevice, s));- patchify_kernel<<<N, 3 * P * P, 0, s>>>(rgb_, width, gw, P, patch_);+ strata_ds4_launch(patchify_kernel, N, 3 * P * P, 0, s, rgb_, width, gw, P, patch_);if (!gemm(h, pe_w_, 3 * P * P, patch_, 3 * P * P, x_, CUDA_R_32F, D, N, 3 * P * P, D, 1.0f, err)) return false;- add_rows_kernel<<<N, kT, 0, s>>>(x_, nullptr, 0, pe_b_, D);+ strata_ds4_launch(add_rows_kernel, N, kT, 0, s, x_, nullptr, 0, pe_b_, D);const float scale = 1.0f / std::sqrt((float) hd);const int qb = (int) std::max<size_t>(1, std::min<size_t>((size_t) N, (size_t) qb_ * max_n_ / (size_t) N));@@ -456,11 +456,11 @@ bool VisionTower::encode(const uint8_t* rgb, int width, int height, std::vector<const uint16_t* wqkv = take();if (wqkv == nullptr) return false;const uint16_t *wo = wqkv + (size_t) H3 * D, *w1 = wo + (size_t) D * D, *w2 = w1 + (size_t) 2 * I * D;- rmsnorm_kernel<<<N, kT, 0, s>>>(x_, n1_[l], D, g.eps, xn_);+ strata_ds4_launch(rmsnorm_kernel, N, kT, 0, s, x_, n1_[l], D, g.eps, xn_);for (int r0 = 0; r0 < N; r0 += rb_) {const int nr = std::min(rb_, N - r0);if (!gemm(h, wqkv, D, xn_ + (size_t) r0 * D, D, y_, CUDA_R_32F, H3, nr, D, H3, 1.0f, err)) return false;- qkv_rope_kernel<<<nr, kT, 0, s>>>(y_, bqkv_[l], r0, gw, inv_freq_, g.heads, hd, qkv_);+ strata_ds4_launch(qkv_rope_kernel, nr, kT, 0, s, y_, bqkv_[l], r0, gw, inv_freq_, g.heads, hd, qkv_);}for (int q0 = 0; q0 < N; q0 += qb) {const int nq = std::min(qb, N - q0);@@ -470,29 +470,29 @@ bool VisionTower::encode(const uint8_t* rgb, int width, int height, std::vector<VB(cublasGemmEx(h, CUBLAS_OP_T, CUBLAS_OP_N, N, nq, hd, &scale, qkv_ + D + hh * hd, CUDA_R_16BF, H3,qkv_ + (size_t) q0 * H3 + hh * hd, CUDA_R_16BF, H3, &zero, s_, CUDA_R_32F, N,CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));- softmax_kernel<<<nq, kSmT, 0, s>>>(s_, N, p_);+ strata_ds4_launch(softmax_kernel, nq, kSmT, 0, s, s_, N, p_);// O^T (hd x queries) = V^T P^T: row-major O [query][head's hd] at column hh * hdVB(cublasGemmEx(h, CUBLAS_OP_N, CUBLAS_OP_N, hd, nq, N, &one, qkv_ + 2 * D + hh * hd, CUDA_R_16BF, H3,p_, CUDA_R_16BF, N, &zero, o_ + hh * hd, CUDA_R_16BF, D, CUBLAS_COMPUTE_32F,CUBLAS_GEMM_DEFAULT));}if (!gemm(h, wo, D, o_, D, y_, CUDA_R_32F, D, nq, D, D, 1.0f, err)) return false;- add_rows_kernel<<<nq, kT, 0, s>>>(x_ + (size_t) q0 * D, y_, D, bo_[l], D);+ strata_ds4_launch(add_rows_kernel, nq, kT, 0, s, x_ + (size_t) q0 * D, y_, D, bo_[l], D);}- rmsnorm_kernel<<<N, kT, 0, s>>>(x_, n2_[l], D, g.eps, xn_);+ strata_ds4_launch(rmsnorm_kernel, N, kT, 0, s, x_, n2_[l], D, g.eps, xn_);for (int r0 = 0; r0 < N; r0 += rb_) {const int nr = std::min(rb_, N - r0);if (!gemm(h, w1, D, xn_ + (size_t) r0 * D, D, y_, CUDA_R_32F, 2 * I, nr, D, 2 * I, 1.0f, err)) return false;- silu_mul_kernel<<<dim3(cdiv(I, kT), nr), kT, 0, s>>>(y_, I, mid_);+ strata_ds4_launch(silu_mul_kernel, dim3(cdiv(I, kT), nr), kT, 0, s, y_, I, mid_);if (!gemm(h, w2, I, mid_, I, y_, CUDA_R_32F, D, nr, I, D, 1.0f, err)) return false;- add_rows_kernel<<<nr, kT, 0, s>>>(x_ + (size_t) r0 * D, y_, D, nullptr, D);+ strata_ds4_launch(add_rows_kernel, nr, kT, 0, s, x_ + (size_t) r0 * D, y_, D, nullptr, D);}done();}- rmsnorm_kernel<<<N, kT, 0, s>>>(x_, norm_, D, g.eps, xn_);+ strata_ds4_launch(rmsnorm_kernel, N, kT, 0, s, x_, norm_, D, g.eps, xn_);// the aligner- unfold_kernel<<<L, kT, 0, s>>>(xn_, gh, gw, M, D, n_w, un_);+ strata_ds4_launch(unfold_kernel, L, kT, 0, s, xn_, gh, gw, M, D, n_w, un_);for (int o0 = 0; o0 < g.out; o0 += r1) {const uint16_t* w = take();if (w == nullptr) return false;@@ -500,7 +500,7 @@ bool VisionTower::encode(const uint8_t* rgb, int width, int height, std::vector<return false;done();}- bias_act_kernel<<<dim3(cdiv(g.out, kT), L), kT, 0, s>>>(y_, al1_b_, g.out, 1, a16_);+ strata_ds4_launch(bias_act_kernel, dim3(cdiv(g.out, kT), L), kT, 0, s, y_, al1_b_, g.out, 1, a16_);for (int o0 = 0; o0 < g.out; o0 += r2) {const uint16_t* w = take();if (w == nullptr) return false;@@ -508,7 +508,7 @@ bool VisionTower::encode(const uint8_t* rgb, int width, int height, std::vector<return false;done();}- bias_act_kernel<<<dim3(cdiv(g.out, kT), L), kT, 0, s>>>(y_, al2_b_, g.out, 0, outb_);+ strata_ds4_launch(bias_act_kernel, dim3(cdiv(g.out, kT), L), kT, 0, s, y_, al2_b_, g.out, 0, outb_);rows.resize((size_t) L * g.out);VK(cudaMemcpyAsync(rows.data(), outb_, rows.size() * 2, cudaMemcpyDeviceToHost, s));VK(cudaStreamSynchronize(s));diff --git a/src/dsv41/vendor/exl3.cu b/src/dsv41/vendor/exl3.cuindex 501010668..4f8ecd26b 100644--- a/src/dsv41/vendor/exl3.cu+++ b/src/dsv41/vendor/exl3.cu@@ -1156,53 +1156,53 @@ bool dispatch(int K, int cb, A&&... args) {}template <int K, int cb> struct RunMv {- static void run(const KBatch& b, int blocks, cudaStream_t s) { mv_kernel<K, cb><<<blocks, kThreads, 0, s>>>(b); }+ static void run(const KBatch& b, int blocks, cudaStream_t s) { strata_ds4_launch(mv_kernel<K, cb>, blocks, kThreads, 0, s, b); }};template <int K, int cb> struct RunWin {static void run(const KWin& a, int groups, cudaStream_t s) {- win_gu_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) groups, 2u), kThreads, 0, s>>>(a);+ strata_ds4_launch(win_gu_kernel<K, cb>, dim3((unsigned) (a.n_ff / kCols), (unsigned) groups, 2u), kThreads, 0, s, a);}};template <int K, int cb> struct RunWinDown {static void run(const KWin& a, int groups, cudaStream_t s) {- win_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, 0, s>>>(a);+ strata_ds4_launch(win_down_kernel<K, cb>, dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, 0, s, a);}};template <int K, int cb> struct RunGU {static void run(const KMoe& a, cudaStream_t s) {- moe_gate_up_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k, 2u), kThreads, 0, s>>>(a);+ strata_ds4_launch(moe_gate_up_kernel<K, cb>, dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k, 2u), kThreads, 0, s, a);}};template <int K, int cb> struct RunDown {static void run(const KMoe& a, cudaStream_t s) {- moe_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);+ strata_ds4_launch(moe_down_kernel<K, cb>, dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s, a);}};template <int K, int cb> struct RunRows {static void run(const Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {- recon_rows_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s>>>(+ strata_ds4_launch(recon_rows_kernel<K, cb>, dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s,m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.suh, m.svh, m.k, dst, bf16);}};#if !defined(STRATA_USE_HIP)template <int K, int cb> struct RunGemm {- static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) { tc::gemm_kernel<K, cb><<<grid, kThreads, 0, s>>>(a); }+ static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) { strata_ds4_launch(tc::gemm_kernel<K, cb>, grid, kThreads, 0, s, a); }};template <int K, int cb> struct RunGateUp {static void run(const tc::GArgs& a, dim3 grid, cudaStream_t s) {- tc::gate_up_kernel<K, cb><<<grid, kThreads, 0, s>>>(a);+ strata_ds4_launch(tc::gate_up_kernel<K, cb>, grid, kThreads, 0, s, a);}};#endiftemplate <int K, int cb> struct RunInnerRows {static void run(const Mat& m, uint16_t* dst, cudaStream_t s) {- inner_rows_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s>>>(+ strata_ds4_launch(inner_rows_kernel<K, cb>, dim3((unsigned) (m.n / kCols), (unsigned) (m.k / kCols)), kThreads, 0, s,m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.k, dst);}};template <int K, int cb> struct RunInner {static void run(const Mat& m, uint16_t* dst, cudaStream_t s) {- recon_inner_kernel<K, cb><<<dim3((unsigned) (m.n / kCols), (unsigned) (m.k / 16)), kThreads, 0, s>>>(+ strata_ds4_launch(recon_inner_kernel<K, cb>, dim3((unsigned) (m.n / kCols), (unsigned) (m.k / 16)), kThreads, 0, s,m.trellis, m.ld > 0 ? m.ld : m.n / 16, m.n, dst);}};@@ -1584,8 +1584,8 @@ bool win_experts(const WinArgs& a, cudaStream_t s) {if (!dispatch<RunWin>(a.lay.K[0], a.lay.cb, w, groups, s) ||!dispatch<RunWinDown>(a.lay.K[2], a.lay.cb, w, groups, s))return false;- if (a.wait_ptr != nullptr) win_wait_kernel<<<1, 32, 0, s>>>(a.wait_ptr, a.wait_val);- win_combine_kernel<<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s>>>(w);+ if (a.wait_ptr != nullptr) strata_ds4_launch(win_wait_kernel, 1, 32, 0, s, a.wait_ptr, a.wait_val);+ strata_ds4_launch(win_combine_kernel, dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s, w);launch_check("win_experts");return true;}@@ -1714,7 +1714,7 @@ void rows_rot_f16(const float* x, int ldx, const int* idx, int nrows, int ncols,cudaStream_t s) {if (nrows <= 0) return;const long long warps = (long long) nrows * (ncols / kCols);- rows_rot_f16_kernel<<<(unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s>>>(x, ldx, idx, nrows, ncols, suh,+ strata_ds4_launch(rows_rot_f16_kernel, (unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s, x, ldx, idx, nrows, ncols, suh,out);launch_check("rows_rot_f16");}@@ -1722,13 +1722,13 @@ void rows_rot_f16(const float* x, int ldx, const int* idx, int nrows, int ncols,void rows_rot_out(float* y, int ldy, int nrows, int ncols, const uint16_t* svh, cudaStream_t s) {if (nrows <= 0) return;const long long warps = (long long) nrows * (ncols / kCols);- rows_rot_out_kernel<<<(unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s>>>(y, ldy, nrows, ncols, svh);+ strata_ds4_launch(rows_rot_out_kernel, (unsigned) ((warps + kWarps - 1) / kWarps), kThreads, 0, s, y, ldy, nrows, ncols, svh);launch_check("rows_rot_out");}void rows_f16(const float* x, int ldx, const int* idx, int nrows, int ncols, uint16_t* out, cudaStream_t s) {if (nrows <= 0) return;- rows_f16_kernel<<<dim3((unsigned) std::min(8, (ncols + 255) / 256), (unsigned) nrows), 256, 0, s>>>(x, ldx, idx,+ strata_ds4_launch(rows_f16_kernel, dim3((unsigned) std::min(8, (ncols + 255) / 256), (unsigned) nrows), 256, 0, s, x, ldx, idx,ncols, out);launch_check("rows_f16");}diff --git a/src/dsv41/vendor/exl3_cpu.cpp b/src/dsv41/vendor/exl3_cpu.cppindex e87efee8b..b7a313fbc 100644--- a/src/dsv41/vendor/exl3_cpu.cpp+++ b/src/dsv41/vendor/exl3_cpu.cpp@@ -1,3 +1,7 @@+#if !defined(__AVX512F__)+// the band GEMM's chunk width, defined here because the uses are outside the AVX-512 guard+constexpr int kGemmK = 256;+#endif// Copyright (c) 2026 Peasant Smith and the Project Maya contributors (MIT; https://github.com/mw00/project-maya, v1.1.17).// Ported into Strata (third_party/project-maya/NOTICE.md, LICENSE); changes are in the git history of this file.// src/kernels/cpu/exl3_cpu.cpp - EXL3 routed experts on the CPU (see include/strata/kernels/cpu/exl3_cpu.hpp).diff --git a/src/dsv41/vendor/exl3_legacy.cu b/src/dsv41/vendor/exl3_legacy.cuindex bcc389b1e..0a028bbda 100644--- a/src/dsv41/vendor/exl3_legacy.cu+++ b/src/dsv41/vendor/exl3_legacy.cu@@ -658,22 +658,22 @@ bool dispatch(int K, int cb, A&&... args) {}template <int K, int cb> struct RunMv {- static void run(const KBatch& b, int blocks, cudaStream_t s) { mv_kernel<K, cb><<<blocks, kThreads, 0, s>>>(b); }+ static void run(const KBatch& b, int blocks, cudaStream_t s) { strata_ds4_launch(mv_kernel<K, cb>, blocks, kThreads, 0, s, b); }};template <int K, int cb> struct RunMvN {static void run(const KBatch& b, int blocks, cudaStream_t s) {- mv_n_kernel<K, cb><<<blocks, kThreads, smem_n_bytes(K), s>>>(b);+ strata_ds4_launch(mv_n_kernel<K, cb>, blocks, kThreads, smem_n_bytes(K), s, b);}};template <int K, int cb> struct RunWin {static void run(const KWin& a, int groups, cudaStream_t s) {- win_gu_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) groups), kThreads, smem_n_bytes(K), s>>>(a);+ strata_ds4_launch(win_gu_kernel<K, cb>, dim3((unsigned) (a.n_ff / kCols), (unsigned) groups), kThreads, smem_n_bytes(K), s, a);}};template <int K, int cb> struct RunWinDown {static void run(const KWin& a, int groups, cudaStream_t s) {- win_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, smem_n_bytes(K),- s>>>(a);+ strata_ds4_launch(win_down_kernel<K, cb>, dim3((unsigned) (a.n_embd / kCols), (unsigned) groups), kThreads, smem_n_bytes(K),+ s, a);}};template <int K, int cb> struct AllowSmemN {@@ -686,12 +686,12 @@ template <int K, int cb> struct AllowSmemN {};template <int K, int cb> struct RunGU {static void run(const KMoe& a, cudaStream_t s) {- moe_gate_up_kernel<K, cb><<<dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);+ strata_ds4_launch(moe_gate_up_kernel<K, cb>, dim3((unsigned) (a.n_ff / kCols), (unsigned) a.k), kThreads, 0, s, a);}};template <int K, int cb> struct RunDown {static void run(const KMoe& a, cudaStream_t s) {- moe_down_kernel<K, cb><<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s>>>(a);+ strata_ds4_launch(moe_down_kernel<K, cb>, dim3((unsigned) (a.n_embd / kCols), (unsigned) a.k), kThreads, 0, s, a);}};// ---------------------------------------------------------------- per-stream scratch@@ -958,7 +958,7 @@ bool win_experts(const WinArgs& a, cudaStream_t s) {if (!dispatch<RunWin>(a.lay.K[0], a.lay.cb, w, groups, s) ||!dispatch<RunWinDown>(a.lay.K[2], a.lay.cb, w, groups, s))return false;- win_combine_kernel<<<dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s>>>(w);+ strata_ds4_launch(win_combine_kernel, dim3((unsigned) (a.n_embd / kCols), (unsigned) a.T), kCols, 0, s, w);launch_check("win_experts");return true;}diff --git a/src/dsv41/vendor/moe_win.cu b/src/dsv41/vendor/moe_win.cuindex fadbad001..1395738d0 100644--- a/src/dsv41/vendor/moe_win.cu+++ b/src/dsv41/vendor/moe_win.cu@@ -47,7 +47,7 @@ __global__ void moe_win_group_kernel(const __grid_constant__ MoeWin w) {} // namespacevoid moe_win_group(const MoeWin& w, cudaStream_t s) {- moe_win_group_kernel<<<1, 32, 0, s>>>(w);+ strata_ds4_launch(moe_win_group_kernel, 1, 32, 0, s, w);const cudaError_t e = cudaGetLastError();if (e != cudaSuccess) {std::fprintf(stderr, "dsv41: moe_win_group: %s\n", cudaGetErrorString(e));diff --git a/src/dsv41/vendor/sampler.cu b/src/dsv41/vendor/sampler.cuindex b406df584..94944c6fb 100644--- a/src/dsv41/vendor/sampler.cu+++ b/src/dsv41/vendor/sampler.cu@@ -525,12 +525,12 @@ void sample_tokens(const float* logits, int n_tokens, int n_vocab, const int* hiif (p.greedy || p.temperature <= 0.0f) {// One block per token, 1,024 threads over the vocabulary. See `sampler_greedy_kernel`.const int gthreads = 1024;- sampler_greedy_kernel<<<(unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream>>>(+ strata_ds4_launch(sampler_greedy_kernel, (unsigned) n_tokens, gthreads, shmem, (cudaStream_t) stream,logits, n_vocab, history, history_len, p, p.penalty_last_n, p.penalty_last_n, out);} else {// The same block-per-token shape: the selection's k argmax rounds reduce inside the block. See// `sampler_kernel`'s header for what the old one-thread-per-token launch cost.- sampler_kernel<0><<<(unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream>>>(+ strata_ds4_launch(sampler_kernel<0>, (unsigned) n_tokens, 1024, shmem, (cudaStream_t) stream,logits, n_vocab, n_tokens, history, history_len, p, out, nullptr, nullptr);}const cudaError_t e = cudaGetLastError();@@ -564,7 +564,7 @@ void sample_draft(const float* logits, int n_tokens, int n_vocab, const SamplerPvoid* stream) {if (n_tokens <= 0) return;spec_check("sample_draft", n_tokens, n_vocab, p);- sampler_kernel<1><<<(unsigned) n_tokens, 1024, 0, (cudaStream_t) stream>>>(logits, n_vocab, n_tokens, nullptr, 0,+ strata_ds4_launch(sampler_kernel<1>, (unsigned) n_tokens, 1024, 0, (cudaStream_t) stream, logits, n_vocab, n_tokens, nullptr, 0,p, out, q, nullptr);spec_launched("sample_draft", stream);}@@ -573,7 +573,7 @@ void sample_verify(const float* logits, int n_tokens, int n_vocab, const Samplerconst int* draft, int* out, void* stream) {if (n_tokens <= 0) return;spec_check("sample_verify", n_tokens, n_vocab, p);- sampler_kernel<2><<<(unsigned) n_tokens, 1024, 0, (cudaStream_t) stream>>>(logits, n_vocab, n_tokens, nullptr, 0,+ strata_ds4_launch(sampler_kernel<2>, (unsigned) n_tokens, 1024, 0, (cudaStream_t) stream, logits, n_vocab, n_tokens, nullptr, 0,p, out, const_cast<SpecList*>(q), draft);spec_launched("sample_verify", stream);}