Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(v4/cuda): the DeepSeek V4 CUDA kernels the engine tier needs — generic sm_80+ + opt-in DeepGEMM, Windows DLL export/loader

dcutugno4eddc628cd61
diff --git a/c/Makefile b/c/Makefile
index 26b97fa5..1113741f 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -479,7 +479,7 @@ CFLAGS += -DCOLI_DSV4_DEEPGEMM
# default g++ is too new for CUDA names a supported host compiler. DeepGEMM
# needs C++20 and the relaxed-constexpr/extended-lambda pair, not its own arch.
override NVCC_STD := c++20
-override NVCCFLAGS += --expt-relaxed-constexpr --expt-extended-lambda -DCOLI_DSV4_DEEPGEMM -I"$(DEEPGEMM_HOME)"
+override NVCCFLAGS += --expt-relaxed-constexpr --expt-extended-lambda -DCOLI_DSV4_DEEPGEMM -I"$(DEEPGEMM_HOME)" -I"$(DEEPGEMM_HOME)/deep_gemm/include" -I"$(DEEPGEMM_HOME)/cutlass/include" -I"$(DEEPGEMM_HOME)/third-party/cutlass/include"
LDFLAGS += -lcuda
endif
ifeq ($(NCCL),1)
@@ -696,6 +696,45 @@ backend_cuda.o: backend_cuda.cu backend_cuda.h backend_gpu_compat.h .build-confi
backend_cuda_dsv4.o: backend_cuda_dsv4.cu backend_cuda_dsv4.h .build-config
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
"$(NVCC)" $(NVCCFLAGS) -c backend_cuda_dsv4.cu -o $@
+
+# Windows DeepSeek V4 CUDA DLLs: the sibling of cuda-dll for the DeepSeek V4
+# engine. deepseek-v4.exe is built with MinGW gcc and cannot link MSVC/nvcc
+# objects, so backend_cuda_dsv4.cu compiles into its own DLL (nvcc + MSVC
+# cl.exe as host) whose symbols are exported via dsv4.def; the engine resolves
+# them at runtime through backend_loader_dsv4.c, mirroring the
+# backend_loader.c / coli_cuda.dll split. Run from a vcvars64 shell, like
+# cuda-dll. Two flavours ship side by side and the loader picks at runtime by
+# compute capability: coli_cuda_dsv4.dll is the GENERIC build (plain CUDA
+# kernels, any sm_80+ card; CUDA_ARCH=portable for a fat binary) and
+# coli_cuda_dsv4_dg.dll the DeepGEMM sm_120a build (tensor-core prefill paths;
+# needs the sm120 DeepGEMM checkout at DEEPGEMM_HOME, see
+# patches-deepgemm-sm120-msvc.patch). Both need -lcuda for the TMA driver API.
+cuda-dsv4-dll: backend_cuda_dsv4.cu backend_cuda_dsv4.h dsv4.def
+ @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
+ @command -v cl >/dev/null 2>&1 || { echo "cl.exe (MSVC) not in PATH — run vcvars64.bat first" >&2; exit 1; }
+ "$(NVCC)" $(NVCCFLAGS) --expt-relaxed-constexpr --expt-extended-lambda -Xcompiler=/Zc:preprocessor -shared \
+ -Xlinker /DEF:dsv4.def \
+ -L"$(CUDA_HOME)/lib/x64" -lcublasLt -lcudart -lcuda \
+ backend_cuda_dsv4.cu -o coli_cuda_dsv4.dll
+
+DEEPGEMM_INC = -I"$(DEEPGEMM_HOME)/deep_gemm/include" -I"$(DEEPGEMM_HOME)/cutlass/include" -I"$(DEEPGEMM_HOME)/third-party/cutlass/include"
+cuda-dsv4-dg-dll: backend_cuda_dsv4.cu backend_cuda_dsv4.h dsv4.def
+ @test -n "$(DEEPGEMM_HOME)" || { echo "DEEPGEMM_HOME must point to the sm120 DeepGEMM checkout" >&2; exit 1; }
+ @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
+ @command -v cl >/dev/null 2>&1 || { echo "cl.exe (MSVC) not in PATH — run vcvars64.bat first" >&2; exit 1; }
+ "$(NVCC)" -O3 -std=c++20 -ftz=false --expt-relaxed-constexpr --expt-extended-lambda -Xcompiler=/Zc:preprocessor \
+ -gencode arch=compute_120a,code=sm_120a -DCOLI_DSV4_DEEPGEMM \
+ $(DEEPGEMM_INC) -shared \
+ -Xlinker /DEF:dsv4.def \
+ -L"$(CUDA_HOME)/lib/x64" -lcublasLt -lcudart -lcuda \
+ backend_cuda_dsv4.cu -o coli_cuda_dsv4_dg.dll
+
+# Windows: the decode-path kernels validated through the runtime DLL loader.
+# deepseek-v4.exe is MinGW gcc; backend_loader_dsv4.c resolves the MSVC/nvcc
+# built coli_cuda_dsv4*.dll and this test proves that ABI boundary.
+dsv4-cuda-loader-test: tests/test_dsv4_decode_cuda.c backend_loader_dsv4.c backend_cuda_dsv4.h
+ $(CC) $(CFLAGS) tests/test_dsv4_decode_cuda.c backend_loader_dsv4.c -o dsv4_decode_loader_test$(EXE)
+ ./dsv4_decode_loader_test$(EXE)
backend_metal.o: backend_metal.mm backend_metal.h
$(METALXX) -c backend_metal.mm -o $@
@@ -994,12 +1033,21 @@ V4_TEST_LINK_OBJS = \
COLI_V4_UNIT_PROMPT.o \
$(addsuffix .o,$(V4_TEST_UNITS)) \
COLI_V4_UNIT_NATIVE_QUANT.o
+# Windows engine objects are compiled with -DCOLI_V4_GPU_TIER (the CUDA tier
+# is resolved from coli_cuda_dsv4*.dll at runtime), so the matvec references in
+# native_quant.h resolve through the GPU unit + the DLL loader. Both are inert
+# without a DLL next to the test binary.
+ifneq ($(IS_WIN),)
+V4_TEST_LINK_OBJS += COLI_V4_UNIT_GPU.o backend_loader_dsv4.o
+V4_TEST_EXTRA_TARGETS = COLI_V4_UNIT_GPU.o backend_loader_dsv4.o
+endif
tests/test_deepseek_v4$(EXE): tests/test_deepseek_v4.c deepseek_v4.c deepseek_v4.h \
Makefile.deepseek-v4.units
$(MAKE) -f Makefile.deepseek-v4 ARCH=$(ARCH) deepseek-v4-test-objs \
COLI_V4_UNIT_ST.o COLI_V4_UNIT_CONFIG.o COLI_V4_UNIT_MATH.o COLI_V4_UNIT_SPARSE_ATTENTION.o \
- COLI_V4_UNIT_RESOURCE_PLAN.o COLI_V4_UNIT_PROMPT.o COLI_V4_UNIT_NATIVE_QUANT.o
+ COLI_V4_UNIT_RESOURCE_PLAN.o COLI_V4_UNIT_PROMPT.o COLI_V4_UNIT_NATIVE_QUANT.o \
+ $(V4_TEST_EXTRA_TARGETS)
$(CC) $(CFLAGS) $< $(V4_TEST_LINK_OBJS) -o $@ -pthread $(LDFLAGS)
V4_OWN_DIR = build/ownership
@@ -1209,7 +1257,7 @@ bench: iobench$(EXE)
# Own line: see the TEST_RULES note above on shared .PHONY lines.
.PHONY: bench-omp-grain
.PHONY: test-asan
-.PHONY: dsv4-cuda-test
+.PHONY: dsv4-cuda-test cuda-dsv4-dll cuda-dsv4-dg-dll dsv4-cuda-loader-test
tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.h
$(CC) $(CFLAGS) tests/test_kv_prefix.c -o tests/test_kv_prefix$(EXE) $(LDFLAGS)
diff --git a/c/backend_cuda_dsv4.cu b/c/backend_cuda_dsv4.cu
index d5bfc52b..d04971b5 100644
--- a/c/backend_cuda_dsv4.cu
+++ b/c/backend_cuda_dsv4.cu
@@ -1,4 +1,5 @@
#include "backend_cuda_dsv4.h"
+#include <chrono>
#include <cuda_runtime.h>
#include <cuda_profiler_api.h>
#include <cuda_bf16.h>
@@ -8,6 +9,12 @@
#include <cstdlib>
#include <cstring>
#include <climits>
+#include <vector>
+#ifdef _WIN32
+#define WIN32_LEAN_AND_MEAN
+#define NOMINMAX
+#include <windows.h>
+#endif
#ifdef COLI_DSV4_NCCL
#include <nccl.h>
#endif
@@ -18,6 +25,17 @@
#include <deep_gemm/impls/sm120_bf16_gemm.cuh>
#endif
+/* The cuBLASLt MXFP8 tensor-core path (tc_plan/tc_fp4_matvec, enabled at
+ * runtime by DSV4_CUDA_TC=1, off by default) needs the cuBLASLt >= 12.8
+ * block-scaling APIs (CUBLASLT_MATMUL_MATRIX_SCALE_VEC32_UE8M0 / *_SCALE_MODE).
+ * Older toolkits (e.g. Jetson JetPack with CUDA 12.6) lack them; build with
+ * -DCOLI_DSV4_NO_TC to compile without that path (generic DLL/object only —
+ * the DeepGEMM sm_120a flavour needs 12.8+ anyway). */
+#if defined(COLI_DSV4_NO_TC)
+#define COLI_DSV4_TC 0
+#else
+#define COLI_DSV4_TC 1
+#endif
struct Dsv4CudaTensor { void *w,*bf16_cache; uint8_t *scale;int *dg_scale; int O,I,fmt,device,own_w,own_scale,own_dg_scale; long long bytes; };
struct Dsv4CudaActivation { float *data; long long elements; int device; };
struct Dsv4CudaGraph { cudaGraph_t graph;cudaGraphExec_t exec;int device; };
@@ -26,7 +44,7 @@ struct MvDesc { const uint8_t *w,*scale; const float *x; float *y; };
struct ExpertPtr { const uint8_t *w,*scale; };
struct Dsv4CudaExpertSet { ExpertPtr *table;int64_t *hash;Dsv4CudaTensor *sg,*su,*sd;uint8_t *bank_fc1,*bank_fc1_scale,*bank_fc2,*bank_fc2_scale;int *bank_fc1_dg_scale,*bank_fc2_dg_scale;int count,device,H,I,vocab,topk; };
struct TcPlan {int O,I,ok;cublasLtMatmulDesc_t op;cublasLtMatrixLayout_t a,b,c,d;cublasLtMatmulHeuristicResult_t algo;};
-struct Dev { int id; cudaStream_t stream,aux; cudaEvent_t ready,fork,join;float *dx,*dy,*p1,*p2,*p3,*p4,*aux1,*aux2,*aux3; size_t xcap,ycap,p1cap,p2cap,p3cap,p4cap,aux1cap,aux2cap,aux3cap; float *hx,*hy;
+struct Dev { int id,sms; cudaStream_t stream,aux; cudaEvent_t ready,fork,join;float *dx,*dy,*p1,*p2,*p3,*p4,*aux1,*aux2,*aux3; size_t xcap,ycap,p1cap,p2cap,p3cap,p4cap,aux1cap,aux2cap,aux3cap; float *hx,*hy;
#ifdef COLI_DSV4_NCCL
ncclComm_t ep_comm;
#endif
@@ -34,7 +52,7 @@ struct Dev { int id; cudaStream_t stream,aux; cudaEvent_t ready,fork,join;float
MvDesc *mvdesc;size_t mvdesccap;float *expert_weights;size_t expert_weightscap;int *expert_ids;size_t expert_idcap;
cublasLtHandle_t lt;void *tc_workspace;uint8_t *tcw,*tcs,*tcx,*tcxs;size_t tcwcap,tcscap,tcxcap,tcxscap;TcPlan plans[2];
#ifdef COLI_DSV4_DEEPGEMM
- uint8_t *dga1,*dga2;int *dgsfa1,*dgsfa2,*dglayout;__nv_bfloat16 *dgmm1,*dgmm2;float *dgref,*dgdensews;size_t dga1cap,dga2cap,dgsfa1cap,dgsfa2cap,dglayoutcap,dgmm1cap,dgmm2cap,dgrefcap,dgdensewscap;
+ uint8_t *dga1,*dga2;int *dgsfa1,*dgsfa2,*dglayout,*dgrowmap;__nv_bfloat16 *dgmm1,*dgmm2;float *dgref,*dgdensews;size_t dga1cap,dga2cap,dgsfa1cap,dgsfa2cap,dglayoutcap,dgrowmapcap,dgmm1cap,dgmm2cap,dgrefcap,dgdensewscap;
#endif
};
static Dev g[16];static int ng;
@@ -46,6 +64,7 @@ static int nok(ncclResult_t e,const char*s){if(e==ncclSuccess)return 1;fprintf(s
#endif
static int buf(void **p,size_t *cap,size_t n){if(*cap>=n)return 1;if(*p)cudaFree(*p);*p=nullptr;*cap=0;if(!ok(cudaMalloc(p,n),"buffer allocation"))return 0;*cap=n;return 1;}
+
__constant__ float e8_table[256];
__global__ void set_decode_state(int*state,int token,int position){if(!threadIdx.x){state[0]=token;state[1]=position;}}
__device__ float e4m3(uint8_t b){int sign=b>>7,e=(b>>3)&15,m=b&7;float v;
@@ -112,6 +131,37 @@ static void run_fp4_grouped(const MvDesc *desc,int count,int O,int I,cudaStream_
else if(rows==2)mv_fp4_grouped<2><<<dim3((O+1)/2,count),256,0,stream>>>(desc,count,O,I);
else mv_fp4_grouped<1><<<dim3(O,count),256,0,stream>>>(desc,count,O,I);
}
+/* Prefill batch GEMM without DeepGEMM: one block per output row and 16-token
+ * tile, so each dequantized weight value is reused across the tile and the
+ * weight matrix is read ceil(T/16) times per call instead of T times (what a
+ * matvec-per-token loop would cost). Numerics match mv<F> exactly: same
+ * e4m3/e8 dequant, same bf16 round-trip for fmt 9. x is tokens-major
+ * [T][I], y is tokens-major [T][O] — the layout dg_dense_batch_dispatch and
+ * the engine's coli_fp8_matmul_batch_ref both use. */
+template<int F> __global__ void mm_batch(const uint8_t *w,const uint8_t *sc,const float *x,float *y,int O,int I,int T){
+ int o=blockIdx.x;if(o>=O)return;int tbase=blockIdx.y*16,tcount=T-tbase<16?T-tbase:16;
+ float sum[16]={};
+ for(int i=threadIdx.x;i<I;i+=blockDim.x){float v;
+ if constexpr(F==8||F==9){v=e4m3(w[(long long)o*I+i])*e8(sc[(o/128)*((I+127)/128)+i/128]);if constexpr(F==9)v=__bfloat162float(__float2bfloat16(v));}
+ else {uint8_t p=w[((long long)o*I+i)>>1],q=(i&1)?p>>4:p&15;v=f4(q)*e8(sc[(long long)o*(I/32)+i/32]);}
+ #pragma unroll
+ for(int t=0;t<16;t++)if(t<tcount)sum[t]+=x[(long long)(tbase+t)*I+i]*v;
+ }
+ __shared__ float s[16][256];
+ #pragma unroll
+ for(int t=0;t<16;t++)s[t][threadIdx.x]=sum[t];
+ __syncthreads();
+ for(int n=128;n;n>>=1){if(threadIdx.x<n){
+ #pragma unroll
+ for(int t=0;t<16;t++)s[t][threadIdx.x]+=s[t][threadIdx.x+n];}__syncthreads();}
+ if(!threadIdx.x)for(int t=0;t<tcount;t++)y[(long long)(tbase+t)*O+o]=s[t][0];
+}
+static void run_mm_batch(Dsv4CudaTensor*t,const float*x,float*y,int T,cudaStream_t stream){
+ dim3 grid(t->O,(T+15)/16);
+ if(t->fmt==4)mm_batch<4><<<grid,256,0,stream>>>((uint8_t*)t->w,t->scale,x,y,t->O,t->I,T);
+ else if(t->fmt==9)mm_batch<9><<<grid,256,0,stream>>>((uint8_t*)t->w,t->scale,x,y,t->O,t->I,T);
+ else mm_batch<8><<<grid,256,0,stream>>>((uint8_t*)t->w,t->scale,x,y,t->O,t->I,T);
+}
__device__ int tc_sf_off(int outer,int inner,int sf_inner_dim){int base=((inner/4)*4+(outer/128)*sf_inner_dim)*128,o=outer&127;return base+(o%32)*16+(o/32)*4+(inner&3);}
__device__ uint8_t e4m3_code(float x){int best=0;float bd=INFINITY;for(int v=0;v<255;v++){float z=e4m3((uint8_t)v),d=fabsf(z-x);if(d<bd||(d==bd&&!(v&1)&&(best&1))){bd=d;best=v;}}return (uint8_t)best;}
__global__ void tc_expand(const uint8_t*q4,const uint8_t*src,uint8_t*q8,uint8_t*dst,int O,int I){
@@ -172,6 +222,10 @@ __global__ void fp8_sim(float *x,int rows,int n){int row=blockIdx.x/((n+127)/128
float q=fmaxf(-448.f,fminf(448.f,x[(long long)row*n+i]/scale));x[(long long)row*n+i]=__bfloat162float(__float2bfloat16(e4m3_round(q)*scale));}}
__global__ void fp8_sim64(float *x,int n){int i=blockIdx.x*64+threadIdx.x;__shared__ float mx[64];float v=i<n?fabsf(x[i]):0;mx[threadIdx.x]=v;__syncthreads();
for(int k=32;k;k>>=1){if(threadIdx.x<k&&mx[threadIdx.x+k]>mx[threadIdx.x])mx[threadIdx.x]=mx[threadIdx.x+k];__syncthreads();}if(i<n){float m=fmaxf(mx[0],1e-4f),raw=m/448.f;int e;frexpf(raw,&e);float scale=ldexpf(1.f,e-1);uint8_t q=__nv_cvt_float_to_fp8(fmaxf(-448.f,fminf(448.f,x[i]/scale)),__NV_SATFINITE,__NV_E4M3);x[i]=__bfloat162float(__float2bfloat16(e4m3(q)*scale));}}
+/* Batched router (generic CUDA; used by both builds). */
+__device__ __forceinline__ bool route_better(float,unsigned,float,unsigned);__device__ __forceinline__ float route_prob(float);
+__global__ void route_logits_batch(const float*w,const float*x,float*y,int tokens,int E,int H){int e=blockIdx.x% E,t=blockIdx.x/E;float sum=0;const float*in=x+(long long)t*H;for(int h=threadIdx.x;h<H;h+=blockDim.x)sum+=w[(long long)e*H+h]*in[h];__shared__ float s[256];s[threadIdx.x]=sum;__syncthreads();for(int n=128;n;n>>=1){if(threadIdx.x<n)s[threadIdx.x]+=s[threadIdx.x+n];__syncthreads();}if(!threadIdx.x)y[(long long)t*E+e]=s[0];(void)tokens;}
+__global__ void route_top6_batch(int*ids,float*weights,const float*bias,const float*logits,int tokens,float scale){int t=blockIdx.x,lane=threadIdx.x;if(t>=tokens||lane>=32)return;const float*src=logits+(long long)t*256;float prob[8],score[8];for(int j=0;j<8;j++){int e=lane+j*32;prob[j]=route_prob(src[e]);score[j]=prob[j]+(bias?bias[e]:0.f);}float outp[6]={};unsigned outi[6]={};for(int k=0;k<6;k++){float bs=-INFINITY,bp=0;unsigned bi=UINT_MAX;for(int j=0;j<8;j++){unsigned e=lane+j*32;if(route_better(score[j],e,bs,bi)){bs=score[j];bp=prob[j];bi=e;}}for(int m=16;m;m>>=1){float os=__shfl_xor_sync(0xffffffff,bs,m),op=__shfl_xor_sync(0xffffffff,bp,m);unsigned oi=__shfl_xor_sync(0xffffffff,bi,m);if(route_better(os,oi,bs,bi)){bs=os;bp=op;bi=oi;}}for(int j=0;j<8;j++)if((unsigned)(lane+j*32)==bi)score[j]=-INFINITY;if(!lane){outp[k]=bp;outi[k]=bi;}}if(!lane){float sum=0;for(int k=0;k<6;k++)sum+=outp[k];sum=fmaxf(sum,6.103515625e-5f);for(int k=0;k<6;k++){ids[(long long)t*6+k]=(int)outi[k];weights[(long long)t*6+k]=outp[k]/sum*scale;}}}
#ifdef COLI_DSV4_DEEPGEMM
__global__ void dg_pack_weight_scale(int*out,const uint8_t*in,int expert,int N,int groups){
int p=blockIdx.x*blockDim.x+threadIdx.x,total=N*(groups/4);if(p>=total)return;int n=p/(groups/4),kg=p%(groups/4);uint32_t v=0;
@@ -255,14 +309,19 @@ __global__ void dg_swiglu_quant_rows(const __nv_bfloat16*in,uint8_t*q,int*sfa,fl
for(int j=0;j<4;j++){int k=base+lane+j*32;q[(long long)row*2048+k]=__nv_cvt_float_to_fp8(fmaxf(-448.f,fminf(448.f,value[j]/scale)),__NV_SATFINITE,__NV_E4M3);}}
__syncthreads();if(threadIdx.x<4){int g=threadIdx.x*4;sfa[(long long)threadIdx.x*rows+row]=(int)((uint32_t)code[g]|((uint32_t)code[g+1]<<8)|((uint32_t)code[g+2]<<16)|((uint32_t)code[g+3]<<24));}}
__global__ void dg_reduce6_weighted(float*out,const __nv_bfloat16*parts,const float*weight,int align){int h=blockIdx.x*blockDim.x+threadIdx.x;if(h<4096){float v=0;for(int e=0;e<6;e++)v+=weight[e]*__bfloat162float(parts[(long long)e*align*4096+h]);out[h]=__bfloat162float(__float2bfloat16(v));}}
-__device__ __forceinline__ bool route_better(float,unsigned,float,unsigned);__device__ __forceinline__ float route_prob(float);
-__global__ void route_logits_batch(const float*w,const float*x,float*y,int tokens,int E,int H){int e=blockIdx.x% E,t=blockIdx.x/E;float sum=0;const float*in=x+(long long)t*H;for(int h=threadIdx.x;h<H;h+=blockDim.x)sum+=w[(long long)e*H+h]*in[h];__shared__ float s[256];s[threadIdx.x]=sum;__syncthreads();for(int n=128;n;n>>=1){if(threadIdx.x<n)s[threadIdx.x]+=s[threadIdx.x+n];__syncthreads();}if(!threadIdx.x)y[(long long)t*E+e]=s[0];(void)tokens;}
-__global__ void route_top6_batch(int*ids,float*weights,const float*bias,const float*logits,int tokens,float scale){int t=blockIdx.x,lane=threadIdx.x;if(t>=tokens||lane>=32)return;const float*src=logits+(long long)t*256;float prob[8],score[8];for(int j=0;j<8;j++){int e=lane+j*32;prob[j]=route_prob(src[e]);score[j]=prob[j]+(bias?bias[e]:0.f);}float outp[6]={};unsigned outi[6]={};for(int k=0;k<6;k++){float bs=-INFINITY,bp=0;unsigned bi=UINT_MAX;for(int j=0;j<8;j++){unsigned e=lane+j*32;if(route_better(score[j],e,bs,bi)){bs=score[j];bp=prob[j];bi=e;}}for(int m=16;m;m>>=1){float os=__shfl_xor_sync(0xffffffff,bs,m),op=__shfl_xor_sync(0xffffffff,bp,m);unsigned oi=__shfl_xor_sync(0xffffffff,bi,m);if(route_better(os,oi,bs,bi)){bs=os;bp=op;bi=oi;}}for(int j=0;j<8;j++)if((unsigned)(lane+j*32)==bi)score[j]=-INFINITY;if(!lane){outp[k]=bp;outi[k]=bi;}}if(!lane){float sum=0;for(int k=0;k<6;k++)sum+=outp[k];sum=fmaxf(sum,6.103515625e-5f);for(int k=0;k<6;k++){ids[(long long)t*6+k]=(int)outi[k];weights[(long long)t*6+k]=outp[k]/sum*scale;}}}
__global__ void route_hash_batch(int*ids,const int64_t*map,const int*tokens,int rows){int p=blockIdx.x*blockDim.x+threadIdx.x;if(p<rows*6)ids[p]=(int)map[(long long)tokens[p/6]*6+p%6];}
__global__ void route_fixed_weights_batch(float*weights,const int*ids,const float*logits,int tokens,float scale){int t=blockIdx.x,k=threadIdx.x;__shared__ float p[6];if(t<tokens&&k<6)p[k]=route_prob(logits[(long long)t*256+ids[(long long)t*6+k]]);__syncthreads();if(t<tokens&&!k){float sum=0;for(int i=0;i<6;i++)sum+=p[i];sum=fmaxf(sum,6.103515625e-5f);for(int i=0;i<6;i++)weights[(long long)t*6+i]=p[i]/sum*scale;}}
__global__ void dg_layout_routes(int*out,const int*ids,int rows,int align){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<rows)out[i]=ids[i/align];}
__global__ void dg_quant_routes(const float*x,uint8_t*q,int*sfa,int tokens,int K,int rows,int align){int route=blockIdx.x/(K/512),pack=blockIdx.x%(K/512);if(route>=tokens*6)return;int token=route/6,row=route*align;__shared__ float mx[128];uint32_t codes=0;for(int b=0;b<4;b++){int k=(pack*4+b)*128+threadIdx.x;float v=x[(long long)token*K+k];mx[threadIdx.x]=fabsf(v);__syncthreads();for(int n=64;n;n>>=1){if(threadIdx.x<n)mx[threadIdx.x]=fmaxf(mx[threadIdx.x],mx[threadIdx.x+n]);__syncthreads();}uint8_t code=dg_scale_code(mx[0]);float scale=dg_scale_value(code);q[(long long)row*K+k]=__nv_cvt_float_to_fp8(fmaxf(-448.f,fminf(448.f,v/scale)),__NV_SATFINITE,__NV_E4M3);if(!threadIdx.x)codes|=(uint32_t)code<<(8*b);__syncthreads();}if(!threadIdx.x)sfa[(long long)pack*rows+row]=(int)codes;}
__global__ void dg_swiglu_quant_routes(const __nv_bfloat16*in,uint8_t*q,int*sfa,const float*weight,float limit,int routes,int rows,int align){int route=blockIdx.x;if(route>=routes)return;int row=route*align;__shared__ float mx[128],value[128];uint32_t pack=0;for(int g=0;g<16;g++){int k=g*128+threadIdx.x;float gate=fminf(__bfloat162float(in[(long long)row*4096+k]),limit),up=fmaxf(-limit,fminf(__bfloat162float(in[(long long)row*4096+2048+k]),limit));float v=__bfloat162float(__float2bfloat16(weight[route]*__bfloat162float(__float2bfloat16((gate/(1.f+expf(-gate)))*up))));value[threadIdx.x]=v;mx[threadIdx.x]=fabsf(v);__syncthreads();for(int n=64;n;n>>=1){if(threadIdx.x<n)mx[threadIdx.x]=fmaxf(mx[threadIdx.x],mx[threadIdx.x+n]);__syncthreads();}uint8_t code=dg_scale_code(mx[0]);float scale=dg_scale_value(code);q[(long long)row*2048+k]=__nv_cvt_float_to_fp8(fmaxf(-448.f,fminf(448.f,value[threadIdx.x]/scale)),__NV_SATFINITE,__NV_E4M3);if(!threadIdx.x){pack|=(uint32_t)code<<(8*(g&3));if((g&3)==3){sfa[(long long)(g/4)*rows+row]=(int)pack;pack=0;}}__syncthreads();}}
+/* EXPERT-GROUPED variants: routes are laid out contiguously per expert (host
+ * builds rowmap[route] -> row and layout[row] -> expert, groups 64-aligned,
+ * pad rows zeroed). Each route's row computes exactly what its old 64x
+ * replicated row did, so results are bitwise the replicated layout with
+ * ~4x fewer GEMM rows. */
+__global__ void dg_quant_routes_map(const float*x,uint8_t*q,int*sfa,int tokens,int K,int rows,const int*rowmap){int route=blockIdx.x/(K/512),pack=blockIdx.x%(K/512);if(route>=tokens*6)return;int token=route/6,row=rowmap[route];__shared__ float mx[128];uint32_t codes=0;for(int b=0;b<4;b++){int k=(pack*4+b)*128+threadIdx.x;float v=x[(long long)token*K+k];mx[threadIdx.x]=fabsf(v);__syncthreads();for(int n=64;n;n>>=1){if(threadIdx.x<n)mx[threadIdx.x]=fmaxf(mx[threadIdx.x],mx[threadIdx.x+n]);__syncthreads();}uint8_t code=dg_scale_code(mx[0]);float scale=dg_scale_value(code);q[(long long)row*K+k]=__nv_cvt_float_to_fp8(fmaxf(-448.f,fminf(448.f,v/scale)),__NV_SATFINITE,__NV_E4M3);if(!threadIdx.x)codes|=(uint32_t)code<<(8*b);__syncthreads();}if(!threadIdx.x)sfa[(long long)pack*rows+row]=(int)codes;}
+__global__ void dg_swiglu_quant_routes_map(const __nv_bfloat16*in,uint8_t*q,int*sfa,const float*weight,float limit,int routes,int rows,const int*rowmap){int route=blockIdx.x;if(route>=routes)return;int row=rowmap[route];__shared__ float mx[128],value[128];uint32_t pack=0;for(int g=0;g<16;g++){int k=g*128+threadIdx.x;float gate=fminf(__bfloat162float(in[(long long)row*4096+k]),limit),up=fmaxf(-limit,fminf(__bfloat162float(in[(long long)row*4096+2048+k]),limit));float v=__bfloat162float(__float2bfloat16(weight[route]*__bfloat162float(__float2bfloat16((gate/(1.f+expf(-gate)))*up))));value[threadIdx.x]=v;mx[threadIdx.x]=fabsf(v);__syncthreads();for(int n=64;n;n>>=1){if(threadIdx.x<n)mx[threadIdx.x]=fmaxf(mx[threadIdx.x],mx[threadIdx.x+n]);__syncthreads();}uint8_t code=dg_scale_code(mx[0]);float scale=dg_scale_value(code);q[(long long)row*2048+k]=__nv_cvt_float_to_fp8(fmaxf(-448.f,fminf(448.f,value[threadIdx.x]/scale)),__NV_SATFINITE,__NV_E4M3);if(!threadIdx.x){pack|=(uint32_t)code<<(8*(g&3));if((g&3)==3){sfa[(long long)(g/4)*rows+row]=(int)pack;pack=0;}}__syncthreads();}}
+__global__ void dg_reduce_routes_map(float*out,const __nv_bfloat16*parts,int tokens,const int*rowmap){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)tokens*4096;if(p<n){int t=p/4096,h=p%4096;float v=0;for(int k=0;k<6;k++)v+=__bfloat162float(parts[(long long)rowmap[t*6+k]*4096+h]);out[p]=__bfloat162float(__float2bfloat16(v));}}
__global__ void dg_reduce_routes(float*out,const __nv_bfloat16*parts,int tokens,int align){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)tokens*4096;if(p<n){int t=p/4096,h=p%4096;float v=0;for(int k=0;k<6;k++)v+=__bfloat162float(parts[((long long)(t*6+k)*align)*4096+h]);out[p]=__bfloat162float(__float2bfloat16(v));}}
__global__ void swiglu_rows(float*out,const float*gate,const float*up,float limit,int rows,int I){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)rows*I;if(p<n){float g=fminf(gate[p],limit),u=fmaxf(-limit,fminf(up[p],limit));out[p]=__bfloat162float(__float2bfloat16((g/(1.f+expf(-g)))*u));}}
__global__ void add_rows(float*out,const float*x,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=__bfloat162float(__float2bfloat16(out[i]+x[i]));}
@@ -298,9 +357,9 @@ static int dg_dense_2048x4096(Dev*c,Dsv4CudaTensor*t,const float*x,float*y){cons
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,t->dg_scale,O,I/512,(unsigned long long)O*4,64,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,4,I/512,16,16,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,O,1,(unsigned long long)O*2,16,64,CU_TENSOR_MAP_SWIZZLE_NONE))return 0;
- auto gemm=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,64,16,64,64,64,0,16,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,64,4>;
+ auto gemm=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,64,16,64,64,64,0,16,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,64,4>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,64,16,64,64,64,0,16,128,256,84,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,64,4>;
if(!ok(cudaFuncSetAttribute(gemm,cudaFuncAttributeMaxDynamicSharedMemorySize,88320),"dense DeepGEMM shared memory"))return 0;
- gemm<<<170,384,88320,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,c->dgdensews,O,1,I,1,O,0,a,b,sa,sb,d);
+ gemm<<<(c->sms>=170?170:84),384,88320,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,c->dgdensews,O,1,I,1,O,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
auto reduce=deep_gemm::sm120_split_k_reduce_impl<DgOutput,4>;reduce<<<(O+255)/256,256,0,c->stream>>>((DgOutput*)c->dgmm1,c->dgdensews,O,1,1,O);
dg_bf16_to_float<<<(O+255)/256,256,0,c->stream>>>(y,c->dgmm1,O);
static int dumped;if(!dumped++&&getenv("DSV4_CUDA_DENSE_DUMP")){struct Item{const char*p;const void*d;size_t n;}items[]={{"/tmp/dsv4-dense-x.f32",x,I*4},{"/tmp/dsv4-dense-q.fp8",c->dga1,I},{"/tmp/dsv4-dense-sfa.bin",c->dgsfa1,(I/512)*4*sizeof(int)},{"/tmp/dsv4-dense-sfb.bin",t->dg_scale,(size_t)O*(I/512)*sizeof(int)},{"/tmp/dsv4-dense-out.f32",y,O*4}};
@@ -315,9 +374,9 @@ static int dg_dense_4096x2048(Dev*c,Dsv4CudaTensor*t,const float*x,float*y){cons
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,t->dg_scale,O,I/512,(unsigned long long)O*4,64,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,4,I/512,16,16,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,O,1,(unsigned long long)O*2,16,64,CU_TENSOR_MAP_SWIZZLE_NONE))return 0;
- auto gemm=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,64,16,64,64,64,0,16,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,64,4>;
+ auto gemm=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,64,16,64,64,64,0,16,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,64,4>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,64,16,64,64,64,0,16,128,256,84,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,64,4>;
if(!ok(cudaFuncSetAttribute(gemm,cudaFuncAttributeMaxDynamicSharedMemorySize,88320),"dense DeepGEMM shared memory"))return 0;
- gemm<<<170,384,88320,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,c->dgdensews,O,1,I,1,O,0,a,b,sa,sb,d);
+ gemm<<<(c->sms>=170?170:84),384,88320,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,c->dgdensews,O,1,I,1,O,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
auto reduce=deep_gemm::sm120_split_k_reduce_impl<DgOutput,4>;reduce<<<(O+255)/256,256,0,c->stream>>>((DgOutput*)c->dgmm1,c->dgdensews,O,1,1,O);
dg_bf16_to_float<<<(O+255)/256,256,0,c->stream>>>(y,c->dgmm1,O);return ok(cudaGetLastError(),"dense DeepGEMM launch");}
template<int O,int I,int BM,int BK,int SW,int STAGES,int EPI,int SPLIT,bool QUANT=true>
@@ -334,9 +393,9 @@ static int dg_dense_attention(Dev*c,Dsv4CudaTensor*t,const float*x,float*y){
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,t->dg_scale,O,I/512,(unsigned long long)O*4,BM,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,4,I/512,16,16,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,O,1,(unsigned long long)O*2,16,BM,CU_TENSOR_MAP_SWIZZLE_NONE)){fprintf(stderr,"[DSV4 DG] attention TMA map failed %dx%d\n",O,I);return 0;}
- auto gemm=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,BM,16,BK,SW,SW,0,STAGES,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,EPI,SPLIT>;
+ auto gemm=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,BM,16,BK,SW,SW,0,STAGES,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,EPI,SPLIT>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,I,128,128,1,BM,16,BK,SW,SW,0,STAGES,128,256,84,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,EPI,SPLIT>;
if(!ok(cudaFuncSetAttribute(gemm,cudaFuncAttributeMaxDynamicSharedMemorySize,SMEM),"attention DeepGEMM shared memory"))return 0;
- gemm<<<170,384,SMEM,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,SPLIT>1?c->dgdensews:nullptr,O,1,I,1,O,0,a,b,sa,sb,d);
+ gemm<<<(c->sms>=170?170:84),384,SMEM,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,SPLIT>1?c->dgdensews:nullptr,O,1,I,1,O,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
if constexpr(SPLIT>1){auto reduce=deep_gemm::sm120_split_k_reduce_impl<DgOutput,SPLIT>;reduce<<<(O+255)/256,256,0,c->stream>>>((DgOutput*)c->dgmm1,c->dgdensews,O,1,1,O);}
dg_bf16_to_float<<<(O+255)/256,256,0,c->stream>>>(y,c->dgmm1,O);return ok(cudaGetLastError(),"attention DeepGEMM launch");
}
@@ -352,9 +411,9 @@ static int dg_dense_batch(Dev*c,Dsv4CudaTensor*t,const float*x,float*y,int token
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,t->dg_scale,O,I/512,(unsigned long long)O*4,BM,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,aligned_tokens,I/512,(unsigned long long)aligned_tokens*4,BN,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,O,tokens,(unsigned long long)O*2,16,BM,CU_TENSOR_MAP_SWIZZLE_NONE))return 0;
- auto gemm=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,0,I,128,128,1,BM,BN,BK,SW,SW,0,STAGES,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,EPI,1>;
+ auto gemm=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,0,I,128,128,1,BM,BN,BK,SW,SW,0,STAGES,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,EPI,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,0,I,128,128,1,BM,BN,BK,SW,SW,0,STAGES,128,256,84,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,false,false,false,true,false,EPI,1>;
if(!ok(cudaFuncSetAttribute(gemm,cudaFuncAttributeMaxDynamicSharedMemorySize,SMEM),"batched dense DeepGEMM shared memory"))return 0;
- gemm<<<170,384,SMEM,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,nullptr,O,tokens,I,1,O,0,a,b,sa,sb,d);
+ gemm<<<(c->sms>=170?170:84),384,SMEM,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,nullptr,O,tokens,I,1,O,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
dg_bf16_to_float<<<((long long)tokens*O+255)/256,256,0,c->stream>>>(y,c->dgmm1,tokens*O);return ok(cudaGetLastError(),"batched dense DeepGEMM launch");
}
static int dg_dense_batch_dispatch(Dev*c,Dsv4CudaTensor*t,const float*x,float*y,int tokens){
@@ -406,9 +465,9 @@ template<int G> static int dg_attention_wo_a_grouped(Dev*c,Dsv4CudaTensor*t,cons
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,t->dg_scale,N,G*(K/512),(unsigned long long)N*4,64,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,4,G*(K/512),16,16,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,N,1,(unsigned long long)N*2,16,64,CU_TENSOR_MAP_SWIZZLE_NONE))return 0;
- auto gemm=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,K,128,128,G,64,16,128,128,128,0,9,128,256,170,deep_gemm::GemmType::Batched,false,DgOutput,DgEpilogue,false,false,false,true,false,64,1>;
+ auto gemm=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,K,128,128,G,64,16,128,128,128,0,9,128,256,170,deep_gemm::GemmType::Batched,false,DgOutput,DgEpilogue,false,false,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,1,K,128,128,G,64,16,128,128,128,0,9,128,256,84,deep_gemm::GemmType::Batched,false,DgOutput,DgEpilogue,false,false,false,true,false,64,1>;
if(!ok(cudaFuncSetAttribute(gemm,cudaFuncAttributeMaxDynamicSharedMemorySize,95872),"wo_a DeepGEMM shared memory"))return 0;
- gemm<<<170,384,95872,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,nullptr,N,1,K,1,N,N,a,b,sa,sb,d);
+ gemm<<<(c->sms>=170?170:84),384,95872,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)t->w,(__nv_fp8_e4m3*)c->dga1,nullptr,nullptr,nullptr,N,1,K,1,N,N,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
dg_bf16_to_float<<<(O+255)/256,256,0,c->stream>>>(y,c->dgmm1,O);return ok(cudaGetLastError(),"wo_a DeepGEMM launch");
}
static int dg_attention_wo_a(Dev*c,Dsv4CudaTensor*t,const float*x,float*y){
@@ -425,9 +484,9 @@ template<int G> static int dg_moe(Dev*c,Dsv4CudaExpertSet*set,const float*weight
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,M,(unsigned long long)G*8,(unsigned long long)M*4,64,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,set->bank_fc1_dg_scale,N,(unsigned long long)G*32,16384,128,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,N,(unsigned long long)G*M,(unsigned long long)N*2,64,64,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto fc1=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,128,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedMasked,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
+ auto fc1=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,128,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedMasked,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,128,128,128,128,128,3,128,256,84,deep_gemm::GemmType::MGroupedMasked,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
if(!ok(cudaFuncSetAttribute(fc1,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"DeepGEMM FC1 shared memory"))return 0;
- fc1<<<170,384,92672,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)c->dga1,(__nv_fp8_e4m3*)set->bank_fc1,c->dglayout,nullptr,nullptr,M,N,4096,N,0,0,a,b,sa,sb,d);
+ fc1<<<(c->sms>=170?170:84),384,92672,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)c->dga1,(__nv_fp8_e4m3*)set->bank_fc1,c->dglayout,nullptr,nullptr,M,N,4096,N,0,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
int debug=getenv("DSV4_CUDA_DG_PROFILE")&&atoi(getenv("DSV4_CUDA_DG_PROFILE"));if(debug&&!ok(cudaStreamSynchronize(c->stream),"DeepGEMM FC1 sync"))return 0;
dg_swiglu_quant_masked<<<6,128,0,c->stream>>>(c->dgmm1,c->dga2,c->dgsfa2,c->expert_ids,weight,limit,M);
if(debug&&!ok(cudaStreamSynchronize(c->stream),"DeepGEMM SwiGLU sync"))return 0;
@@ -436,9 +495,9 @@ template<int G> static int dg_moe(Dev*c,Dsv4CudaExpertSet*set,const float*weight
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa2,M,(unsigned long long)G*4,(unsigned long long)M*4,64,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,set->bank_fc2_dg_scale,N,(unsigned long long)G*16,16384,128,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm2,N,(unsigned long long)G*M,(unsigned long long)N*2,64,64,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto fc2=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,128,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedMasked,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
+ auto fc2=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,128,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedMasked,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,128,128,128,128,128,3,128,256,84,deep_gemm::GemmType::MGroupedMasked,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
if(!ok(cudaFuncSetAttribute(fc2,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"DeepGEMM FC2 shared memory"))return 0;
- fc2<<<170,384,92672,c->stream>>>((DgOutput*)c->dgmm2,nullptr,(__nv_fp8_e4m3*)c->dga2,(__nv_fp8_e4m3*)set->bank_fc2,c->dglayout,nullptr,nullptr,M,N,2048,N,0,0,a,b,sa,sb,d);
+ fc2<<<(c->sms>=170?170:84),384,92672,c->stream>>>((DgOutput*)c->dgmm2,nullptr,(__nv_fp8_e4m3*)c->dga2,(__nv_fp8_e4m3*)set->bank_fc2,c->dglayout,nullptr,nullptr,M,N,2048,N,0,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
if(debug&&!ok(cudaStreamSynchronize(c->stream),"DeepGEMM FC2 sync"))return 0;dg_reduce6_masked<<<16,256,0,c->stream>>>(out,c->dgmm2,c->expert_ids,M);if(debug&&!ok(cudaStreamSynchronize(c->stream),"DeepGEMM reduce sync"))return 0;return ok(cudaGetLastError(),"DeepGEMM routed MoE launch");
}
template<int G> static int dg_moe_contiguous_fast(Dev*c,Dsv4CudaExpertSet*set,const float*weight,float limit,float*out,int ep_base=-1){constexpr int M=384,N=4096,A=64;
@@ -453,9 +512,9 @@ template<int G> static int dg_moe_contiguous_fast(Dev*c,Dsv4CudaExpertSet*set,co
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,M,8,(unsigned long long)M*4,A,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,set->bank_fc1_dg_scale,N,(unsigned long long)G*32,16384,64,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,N,M,(unsigned long long)N*2,64,A,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto fc1=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,64,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
+ auto fc1=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,64,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,64,128,128,128,128,3,128,256,84,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
if(!ok(cudaFuncSetAttribute(fc1,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"DeepGEMM FC1 shared memory"))return 0;
- fc1<<<170,384,92672,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)c->dga1,(__nv_fp8_e4m3*)set->bank_fc1,c->dglayout,nullptr,nullptr,M,N,4096,N,0,0,a,b,sa,sb,d);
+ fc1<<<(c->sms>=170?170:84),384,92672,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)c->dga1,(__nv_fp8_e4m3*)set->bank_fc1,c->dglayout,nullptr,nullptr,M,N,4096,N,0,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
if(ep_base>=0)dg_swiglu_quant_unique_ep<<<6,128,0,c->stream>>>(c->dgmm1,c->dga2,c->dgsfa2,c->expert_ids,ep_base,limit,M,A);
else dg_swiglu_quant_unique<<<6,128,0,c->stream>>>(c->dgmm1,c->dga2,c->dgsfa2,weight,limit,M,A);
dg_replicate_expert_rows<<<((long long)M*2048+255)/256,256,0,c->stream>>>(c->dga2,c->dgsfa2,2048,M,A);
@@ -464,20 +523,47 @@ template<int G> static int dg_moe_contiguous_fast(Dev*c,Dsv4CudaExpertSet*set,co
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa2,M,4,(unsigned long long)M*4,A,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,set->bank_fc2_dg_scale,N,(unsigned long long)G*16,16384,128,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm2,N,M,(unsigned long long)N*2,64,A,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto fc2=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,128,128,128,128,128,2,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
+ auto fc2=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,128,128,128,128,128,2,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,128,128,128,128,128,2,128,256,84,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
if(!ok(cudaFuncSetAttribute(fc2,cudaFuncAttributeMaxDynamicSharedMemorySize,101376),"DeepGEMM FC2 shared memory"))return 0;
- fc2<<<170,384,101376,c->stream>>>((DgOutput*)c->dgmm2,nullptr,(__nv_fp8_e4m3*)c->dga2,(__nv_fp8_e4m3*)set->bank_fc2,c->dglayout,nullptr,nullptr,M,N,2048,N,0,0,a,b,sa,sb,d);
+ fc2<<<(c->sms>=170?170:84),384,101376,c->stream>>>((DgOutput*)c->dgmm2,nullptr,(__nv_fp8_e4m3*)c->dga2,(__nv_fp8_e4m3*)set->bank_fc2,c->dglayout,nullptr,nullptr,M,N,2048,N,0,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
if(ep_base>=0)dg_reduce6_ep<<<16,256,0,c->stream>>>(out,c->dgmm2,c->expert_ids,weight,ep_base,A);else dg_reduce6<<<16,256,0,c->stream>>>(out,c->dgmm2,A);
return ok(cudaGetLastError(),"DeepGEMM routed MoE fast launch");
}
-static int dg_moe_batch_contiguous(Dev*c,Dsv4CudaExpertSet*set,const float*input,const int*ids,const float*weight,int tokens,float limit,float*out){constexpr int N=4096,A=64,G=256;int routes=tokens*6,M=routes*A;size_t a1=(size_t)M*4096,a2=(size_t)M*2048,s1=(size_t)M*8*4,s2=(size_t)M*4*4,mm=(size_t)M*N*2;
+/* ids/weight are DEVICE pointers (uploaded by the caller); host_ids is the
+ * host copy used to build the expert-grouped layout (nullptr = legacy
+ * replicated layout, also DSV4_CUDA_MOE_GROUPED=0). */
+static int dg_moe_batch_contiguous(Dev*c,Dsv4CudaExpertSet*set,const float*input,const int*ids,const float*weight,int tokens,float limit,float*out,const int*host_ids=nullptr){constexpr int N=4096,A=64,G=256;int routes=tokens*6;
+ static int grouped=-1;if(grouped<0){const char*e=getenv("DSV4_CUDA_MOE_GROUPED");grouped=!(e&&*e=='0');}
+ int M=routes*A;
+ std::vector<int> hrow,hlayout;
+ if(host_ids&&grouped){
+ /* Stable bucket by expert: rows per expert padded to A. */
+ int cnt[G]={0};for(int r=0;r<routes;r++){int e=host_ids[r];if(e<0||e>=G)return 0;cnt[e]++;}
+ int base[G];int m=0;for(int e=0;e<G;e++){base[e]=m;if(cnt[e])m+=(cnt[e]+A-1)/A*A;}
+ M=m;hrow.resize(routes);hlayout.assign(M,0);
+ int fill[G]={0};for(int r=0;r<routes;r++){int e=host_ids[r];hrow[r]=base[e]+fill[e]++;}
+ for(int e=0;e<G;e++)if(cnt[e]){int g=(cnt[e]+A-1)/A*A;for(int i=0;i<g;i++)hlayout[base[e]+i]=e;}
+ }
+ size_t a1=(size_t)M*4096,a2=(size_t)M*2048,s1=(size_t)M*8*4,s2=(size_t)M*4*4,mm=(size_t)M*N*2;
if(set->count!=G||set->H!=4096||set->I!=2048||!buf((void**)&c->dga1,&c->dga1cap,a1)||!buf((void**)&c->dga2,&c->dga2cap,a2)||!buf((void**)&c->dgsfa1,&c->dgsfa1cap,s1)||!buf((void**)&c->dgsfa2,&c->dgsfa2cap,s2)||!buf((void**)&c->dglayout,&c->dglayoutcap,(size_t)M*sizeof(int))||!buf((void**)&c->dgmm1,&c->dgmm1cap,mm)||!buf((void**)&c->dgmm2,&c->dgmm2cap,mm))return 0;
- dg_layout_routes<<<(M+255)/256,256,0,c->stream>>>(c->dglayout,ids,M,A);dg_quant_routes<<<routes*8,128,0,c->stream>>>(input,c->dga1,c->dgsfa1,tokens,4096,M,A);dg_replicate_expert_rows<<<((long long)M*4096+255)/256,256,0,c->stream>>>(c->dga1,c->dgsfa1,4096,M,A);CUtensorMap a,b,sa,sb,d;
+ bool use_map=!hrow.empty();
+ if(use_map){
+ if(!buf((void**)&c->dgrowmap,&c->dgrowmapcap,(size_t)routes*sizeof(int))||
+ !ok(cudaMemcpyAsync(c->dgrowmap,hrow.data(),(size_t)routes*sizeof(int),cudaMemcpyHostToDevice,c->stream),"grouped MoE rowmap upload")||
+ !ok(cudaMemcpyAsync(c->dglayout,hlayout.data(),(size_t)M*sizeof(int),cudaMemcpyHostToDevice,c->stream),"grouped MoE layout upload")||
+ !ok(cudaMemsetAsync(c->dga1,0,a1,c->stream),"grouped MoE A1 clear")||!ok(cudaMemsetAsync(c->dgsfa1,0,s1,c->stream),"grouped MoE S1 clear")||
+ !ok(cudaMemsetAsync(c->dga2,0,a2,c->stream),"grouped MoE A2 clear")||!ok(cudaMemsetAsync(c->dgsfa2,0,s2,c->stream),"grouped MoE S2 clear"))return 0;
+ dg_quant_routes_map<<<routes*8,128,0,c->stream>>>(input,c->dga1,c->dgsfa1,tokens,4096,M,c->dgrowmap);
+ }else{
+ dg_layout_routes<<<(M+255)/256,256,0,c->stream>>>(c->dglayout,ids,M,A);dg_quant_routes<<<routes*8,128,0,c->stream>>>(input,c->dga1,c->dgsfa1,tokens,4096,M,A);dg_replicate_expert_rows<<<((long long)M*4096+255)/256,256,0,c->stream>>>(c->dga1,c->dgsfa1,4096,M,A);
+ }
+ CUtensorMap a,b,sa,sb,d;
if(!dg_map(&a,CU_TENSOR_MAP_DATA_TYPE_UINT8,c->dga1,4096,M,4096,128,A,CU_TENSOR_MAP_SWIZZLE_128B)||!dg_map(&b,CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B,set->bank_fc1,4096,(unsigned long long)G*N,2048,128,64,CU_TENSOR_MAP_SWIZZLE_128B)||!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,M,8,(unsigned long long)M*4,A,1,CU_TENSOR_MAP_SWIZZLE_NONE)||!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,set->bank_fc1_dg_scale,N,(unsigned long long)G*32,16384,64,1,CU_TENSOR_MAP_SWIZZLE_NONE)||!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,N,M,(unsigned long long)N*2,64,A,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto fc1=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,64,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;if(!ok(cudaFuncSetAttribute(fc1,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"batched MoE FC1 shared memory"))return 0;fc1<<<170,384,92672,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)c->dga1,(__nv_fp8_e4m3*)set->bank_fc1,c->dglayout,nullptr,nullptr,M,N,4096,N,0,0,a,b,sa,sb,d);
- dg_swiglu_quant_routes<<<routes,128,0,c->stream>>>(c->dgmm1,c->dga2,c->dgsfa2,weight,limit,routes,M,A);dg_replicate_expert_rows<<<((long long)M*2048+255)/256,256,0,c->stream>>>(c->dga2,c->dgsfa2,2048,M,A);
+ auto fc1=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,64,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,4096,128,32,G,64,64,128,128,128,128,3,128,256,84,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;if(!ok(cudaFuncSetAttribute(fc1,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"batched MoE FC1 shared memory"))return 0;fc1<<<(c->sms>=170?170:84),384,92672,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)c->dga1,(__nv_fp8_e4m3*)set->bank_fc1,c->dglayout,nullptr,nullptr,M,N,4096,N,0,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
+ if(use_map)dg_swiglu_quant_routes_map<<<routes,128,0,c->stream>>>(c->dgmm1,c->dga2,c->dgsfa2,weight,limit,routes,M,c->dgrowmap);
+ else{dg_swiglu_quant_routes<<<routes,128,0,c->stream>>>(c->dgmm1,c->dga2,c->dgsfa2,weight,limit,routes,M,A);dg_replicate_expert_rows<<<((long long)M*2048+255)/256,256,0,c->stream>>>(c->dga2,c->dgsfa2,2048,M,A);}
if(!dg_map(&a,CU_TENSOR_MAP_DATA_TYPE_UINT8,c->dga2,2048,M,2048,128,A,CU_TENSOR_MAP_SWIZZLE_128B)||!dg_map(&b,CU_TENSOR_MAP_DATA_TYPE_16U4_ALIGN16B,set->bank_fc2,2048,(unsigned long long)G*N,1024,128,64,CU_TENSOR_MAP_SWIZZLE_128B)||!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa2,M,4,(unsigned long long)M*4,A,1,CU_TENSOR_MAP_SWIZZLE_NONE)||!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,set->bank_fc2_dg_scale,N,(unsigned long long)G*16,16384,64,1,CU_TENSOR_MAP_SWIZZLE_NONE)||!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm2,N,M,(unsigned long long)N*2,64,A,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto fc2=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,64,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;if(!ok(cudaFuncSetAttribute(fc2,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"batched MoE FC2 shared memory"))return 0;fc2<<<170,384,92672,c->stream>>>((DgOutput*)c->dgmm2,nullptr,(__nv_fp8_e4m3*)c->dga2,(__nv_fp8_e4m3*)set->bank_fc2,c->dglayout,nullptr,nullptr,M,N,2048,N,0,0,a,b,sa,sb,d);dg_reduce_routes<<<((long long)tokens*4096+255)/256,256,0,c->stream>>>(out,c->dgmm2,tokens,A);return ok(cudaGetLastError(),"batched contiguous MoE launch");
+ auto fc2=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,64,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N,2048,128,32,G,64,64,128,128,128,128,3,128,256,84,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;if(!ok(cudaFuncSetAttribute(fc2,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"batched MoE FC2 shared memory"))return 0;fc2<<<(c->sms>=170?170:84),384,92672,c->stream>>>((DgOutput*)c->dgmm2,nullptr,(__nv_fp8_e4m3*)c->dga2,(__nv_fp8_e4m3*)set->bank_fc2,c->dglayout,nullptr,nullptr,M,N,2048,N,0,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);if(use_map)dg_reduce_routes_map<<<((long long)tokens*4096+255)/256,256,0,c->stream>>>(out,c->dgmm2,tokens,c->dgrowmap);else dg_reduce_routes<<<((long long)tokens*4096+255)/256,256,0,c->stream>>>(out,c->dgmm2,tokens,A);return ok(cudaGetLastError(),"batched contiguous MoE launch");
}
static int dg_moe_tp2(Dev*c,Dsv4CudaExpertSet*set,const float*weight,float limit,float*out){constexpr int G=256,M=384,A=64,H=4096,J=1024,N1=2048;
int trace=getenv("DSV4_CUDA_TP2_TRACE")&&atoi(getenv("DSV4_CUDA_TP2_TRACE"));
@@ -495,9 +581,9 @@ static int dg_moe_tp2(Dev*c,Dsv4CudaExpertSet*set,const float*weight,float limit
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa1,M,H/512,(unsigned long long)M*4,A,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,set->bank_fc1_dg_scale,N1,(unsigned long long)G*(H/128),(unsigned long long)N1*4,128,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,N1,M,(unsigned long long)N1*2,64,A,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto fc1=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N1,H,128,32,G,64,128,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
+ auto fc1=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N1,H,128,32,G,64,128,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,N1,H,128,32,G,64,128,128,128,128,128,3,128,256,84,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
if(!ok(cudaFuncSetAttribute(fc1,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"TP2 DeepGEMM FC1 shared memory"))return 0;
- fc1<<<170,384,92672,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)c->dga1,(__nv_fp8_e4m3*)set->bank_fc1,c->dglayout,nullptr,nullptr,M,N1,H,N1,0,0,a,b,sa,sb,d);
+ fc1<<<(c->sms>=170?170:84),384,92672,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(__nv_fp8_e4m3*)c->dga1,(__nv_fp8_e4m3*)set->bank_fc1,c->dglayout,nullptr,nullptr,M,N1,H,N1,0,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
if(trace){fprintf(stderr,"[DSV4 TP2] device %d FC1 launched\n",c->id);fflush(stderr);if(!ok(cudaStreamSynchronize(c->stream),"TP2 FC1 trace sync"))return 0;fprintf(stderr,"[DSV4 TP2] device %d FC1 done\n",c->id);fflush(stderr);}
dg_swiglu_quant_unique_tp2<<<6,128,0,c->stream>>>(c->dgmm1,c->dga2,c->dgsfa2,limit,M,A);
dg_replicate_expert_rows<<<((long long)M*J+255)/256,256,0,c->stream>>>(c->dga2,c->dgsfa2,J,M,A);
@@ -507,9 +593,9 @@ static int dg_moe_tp2(Dev*c,Dsv4CudaExpertSet*set,const float*weight,float limit
!dg_map(&sa,CU_TENSOR_MAP_DATA_TYPE_INT32,c->dgsfa2,M,J/512,(unsigned long long)M*4,A,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&sb,CU_TENSOR_MAP_DATA_TYPE_INT32,set->bank_fc2_dg_scale,H,(unsigned long long)G*(J/128),(unsigned long long)H*4,128,1,CU_TENSOR_MAP_SWIZZLE_NONE)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm2,H,M,(unsigned long long)H*2,64,A,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto fc2=deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,H,J,128,32,G,64,128,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
+ auto fc2=c->sms>=170?deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,H,J,128,32,G,64,128,128,128,128,128,3,128,256,170,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>:deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl<0,H,J,128,32,G,64,128,128,128,128,128,3,128,256,84,deep_gemm::GemmType::MGroupedContiguous,false,DgOutput,DgEpilogue,false,true,false,true,false,64,1>;
if(!ok(cudaFuncSetAttribute(fc2,cudaFuncAttributeMaxDynamicSharedMemorySize,92672),"TP2 DeepGEMM FC2 shared memory"))return 0;
- fc2<<<170,384,92672,c->stream>>>((DgOutput*)c->dgmm2,nullptr,(__nv_fp8_e4m3*)c->dga2,(__nv_fp8_e4m3*)set->bank_fc2,c->dglayout,nullptr,nullptr,M,H,J,H,0,0,a,b,sa,sb,d);
+ fc2<<<(c->sms>=170?170:84),384,92672,c->stream>>>((DgOutput*)c->dgmm2,nullptr,(__nv_fp8_e4m3*)c->dga2,(__nv_fp8_e4m3*)set->bank_fc2,c->dglayout,nullptr,nullptr,M,H,J,H,0,0,deep_gemm::TmaParamPad48{},a,b,sa,sb,d);
if(trace){fprintf(stderr,"[DSV4 TP2] device %d FC2 launched\n",c->id);fflush(stderr);if(!ok(cudaStreamSynchronize(c->stream),"TP2 FC2 trace sync"))return 0;fprintf(stderr,"[DSV4 TP2] device %d FC2 done\n",c->id);fflush(stderr);}
dg_reduce6_weighted<<<16,256,0,c->stream>>>(out,c->dgmm2,weight,A);
return ok(cudaGetLastError(),"TP2 DeepGEMM routed MoE launch");
@@ -526,6 +612,14 @@ static int dg_dump_once(Dev*c){static int dumped;if(dumped++||!(getenv("DSV4_CUD
#endif
__global__ void expert_reduce(float *y,const float *parts,int count,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){float v=0;
for(int e=0;e<count;e++)v+=__bfloat162float(__float2bfloat16(parts[(long long)e*n+i]));y[i]=v;}}
+/* Batched siblings of expert_reduce / expert_act(weight=1) for the generic
+ * batched MoE: parts hold `count` rows per token ([t*count+e][n]). */
+__global__ void expert_reduce_rows(float *y,const float *parts,int tokens,int count,int n){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x;
+ if(p<(long long)tokens*n){int t=p/n,i=p%n;float v=0;for(int e=0;e<count;e++)v+=__bfloat162float(__float2bfloat16(parts[((long long)t*count+e)*n+i]));y[p]=v;}}
+__global__ void expert_act_rows1(float *a,const float *g,const float *u,float limit,long long n){long long i=(long long)blockIdx.x*blockDim.x+threadIdx.x;if(i<n){
+ float gg=fminf(__bfloat162float(__float2bfloat16(g[i])),limit);
+ float uu=fmaxf(-limit,fminf(__bfloat162float(__float2bfloat16(u[i])),limit));
+ a[i]=__bfloat162float(__float2bfloat16((gg/(1.f+expf(-gg)))*uu));}}
__global__ void moe_combine(float *routed,const float *shared,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)
routed[i]=__bfloat162float(__float2bfloat16(routed[i]+__bfloat162float(__float2bfloat16(shared[i]))));}
__global__ void ep2_localize(int *ids,float *weights,int base){int k=threadIdx.x;if(k<6){int id=ids[k]-base;if(id<0||id>=128){ids[k]=-1;weights[k]=0.f;}else ids[k]=id;}}
@@ -556,10 +650,469 @@ __global__ void attention_window_kernel(float *context,const float *q,const floa
int h=blockIdx.x;if(h>=heads)return;extern __shared__ float score[];__shared__ float mx,den;int start=pos-window+1;if(start<0)start=0;int nw=pos-start+1;
if(!threadIdx.x){mx=sink[h];for(int t=start;t<=pos;t++){float z=0,*v=(float*)cache+(long long)(t%window)*dim;for(int d=0;d<dim;d++)z+=q[(long long)h*dim+d]*v[d];z/=sqrtf((float)dim);score[t-start]=z;mx=fmaxf(mx,z);}for(int i=0;i<cn;i++){float z=0,*v=(float*)compressed+(long long)i*dim;for(int d=0;d<dim;d++)z+=q[(long long)h*dim+d]*v[d];z/=sqrtf((float)dim);score[nw+i]=z;mx=fmaxf(mx,z);}den=expf(sink[h]-mx);for(int i=0;i<nw+cn;i++){float a=expf(score[i]-mx);score[i]=a;den+=a;}}
__syncthreads();for(int d=threadIdx.x;d<dim;d+=blockDim.x){float sum=0;for(int t=start;t<=pos;t++)sum+=score[t-start]*cache[((long long)(t%window))*dim+d];for(int i=0;i<cn;i++)sum+=score[nw+i]*compressed[(long long)i*dim+d];context[(long long)h*dim+d]=__bfloat162float(__float2bfloat16(sum/den));}}
+/* Prefill batched sparse window attention. vals is a linear slab of KV rows:
+ * [0, comp_base) window rows in position order, [comp_base, ...) the
+ * compressed rows. meta[3t] = first window row for token t, meta[3t+1] = its
+ * window row count, meta[3t+2] = its visible compressed prefix. Numerics
+ * mirror coli_v4_sparse_attention_ref: fp32 scores, max over scores only
+ * (sink joins the denominator, not the max), probabilities rounded to bf16
+ * before the value sum, output rounded to bf16 after the division. */
+/* ---- persistent per-layer KV cache for batched prefill attention ----
+ * The engine rebuilt and re-uploaded the linear KV slab every chunk-layer
+ * (~2 GB PCIe per 826-token prefill). Instead: a per-layer device RING of
+ * window rows (absolute position % window) appended once per chunk, plus an
+ * append-only compressed-prefix buffer. The engine tracks validity by exact
+ * position continuity and reseeds on any mismatch, so no invalidation hooks
+ * are needed here. */
+#define DSV4_KV_CACHE_LAYERS 64
+static float *g_kv_ring[DSV4_KV_CACHE_LAYERS];
+static int g_kv_ring_rows[DSV4_KV_CACHE_LAYERS];
+static float *g_kv_comp[DSV4_KV_CACHE_LAYERS];
+static int g_kv_comp_cap[DSV4_KV_CACHE_LAYERS];
+
+/* Which GPUs this build's kernels can run on. The DeepGEMM build carries
+ * sm_120a-only block-scaled MMA kernels; the generic build is a portable fat
+ * binary of plain CUDA kernels (fp32 compute from fp8/fp4 decode) that runs
+ * on any sm_80+ card. The engine's loader tries the DeepGEMM DLL first and
+ * falls back to the generic one when this says no. */
+extern "C" int dsv4_cuda_backend_arch_ok(int device){
+ cudaDeviceProp prop;
+ if(cudaGetDeviceProperties(&prop,device)!=cudaSuccess){cudaGetLastError();return 0;}
+#ifdef COLI_DSV4_DEEPGEMM
+ return prop.major==12;
+#else
+ return prop.major>=8;
+#endif
+}
+extern "C" const char *dsv4_cuda_backend_name(void){
+#ifdef COLI_DSV4_DEEPGEMM
+ return "deepgemm-sm120";
+#else
+ return "generic";
+#endif
+}
+
+extern "C" long long dsv4_cuda_mem_free_mb(int device){
+ if(!ok(cudaSetDevice(device),"select meminfo device"))return -1;
+ size_t free_b=0,total_b=0;
+ if(!ok(cudaMemGetInfo(&free_b,&total_b),"device meminfo"))return -1;
+ return (long long)(free_b>>20);
+}
+
+extern "C" int dsv4_cuda_kv_ring_append(int device,int layer,const float*rows,
+ int start_pos,int count,int window,int dim){
+ Dev*c=ctx(device);
+ if(!c||layer<0||layer>=DSV4_KV_CACHE_LAYERS||!rows||start_pos<0||count<1||
+ window<1||dim<1||!ok(cudaSetDevice(device),"select kv ring device"))return 0;
+ if(g_kv_ring[layer]&&g_kv_ring_rows[layer]!=window){cudaFree(g_kv_ring[layer]);g_kv_ring[layer]=NULL;}
+ if(!g_kv_ring[layer]){
+ if(!ok(cudaMalloc((void**)&g_kv_ring[layer],(size_t)window*dim*4),"kv ring allocation"))return 0;
+ g_kv_ring_rows[layer]=window;
+ }
+ int done=0;
+ while(done<count){
+ int slot=(start_pos+done)%window;
+ int run=count-done;if(run>window-slot)run=window-slot;
+ if(!ok(cudaMemcpyAsync(g_kv_ring[layer]+(size_t)slot*dim,rows+(size_t)done*dim,
+ (size_t)run*dim*4,cudaMemcpyHostToDevice,c->stream),"kv ring append"))return 0;
+ done+=run;
+ }
+ return 1;
+}
+
+extern "C" int dsv4_cuda_kv_comp_append(int device,int layer,const float*rows,
+ int start_idx,int count,int dim){
+ Dev*c=ctx(device);
+ if(!c||layer<0||layer>=DSV4_KV_CACHE_LAYERS||!rows||start_idx<0||count<1||dim<1||
+ !ok(cudaSetDevice(device),"select kv comp device"))return 0;
+ int need=start_idx+count;
+ if(g_kv_comp_cap[layer]<need){
+ int cap=g_kv_comp_cap[layer]?g_kv_comp_cap[layer]:512;
+ while(cap<need)cap*=2;
+ float*grown=NULL;
+ if(!ok(cudaMalloc((void**)&grown,(size_t)cap*dim*4),"kv comp allocation"))return 0;
+ if(g_kv_comp[layer]&&start_idx&&
+ !ok(cudaMemcpyAsync(grown,g_kv_comp[layer],(size_t)start_idx*dim*4,
+ cudaMemcpyDeviceToDevice,c->stream),"kv comp grow copy")){cudaFree(grown);return 0;}
+ if(g_kv_comp[layer]){cudaStreamSynchronize(c->stream);cudaFree(g_kv_comp[layer]);}
+ g_kv_comp[layer]=grown;g_kv_comp_cap[layer]=cap;
+ }
+ return ok(cudaMemcpyAsync(g_kv_comp[layer]+(size_t)start_idx*dim,rows,
+ (size_t)count*dim*4,cudaMemcpyHostToDevice,c->stream),"kv comp append");
+}
+
+/* Window rows at absolute position >= chunk_start come from the chunk buffer,
+ * not the ring: with the ring indexed position%window, appending the chunk
+ * before attention would let its newest rows overwrite the oldest window rows
+ * that early chunk tokens still need (only bites once start+batch > window).
+ * The engine appends the chunk to the ring AFTER attention instead. */
+__device__ __forceinline__ const float*cached_row(const float*win,const float*chunk,
+ int ap,int chunk_start,int window,int dim){
+ return ap>=chunk_start?chunk+(long long)(ap-chunk_start)*dim
+ :win+(long long)(ap%window)*dim;
+}
+
+__global__ void sparse_attn_batch_cached_kernel(float*out,const float*q,const float*win,
+ const float*chunk,const float*comp,const float*sink,const int*meta,int abs_base,
+ int chunk_start,int window,int heads,int dim,float scale){
+ int t=blockIdx.y,h=blockIdx.x;
+ int wo=meta[3*t],wn=meta[3*t+1],cn=meta[3*t+2],total=wn+cn;
+ const float*qh=q+((long long)t*heads+h)*dim;
+ extern __shared__ float score[];
+ __shared__ float den;
+ int warp=threadIdx.x>>5,lane=threadIdx.x&31;
+ for(int item=warp;item<total;item+=blockDim.x/32){
+ const float*v=item<wn?cached_row(win,chunk,abs_base+wo+item,chunk_start,window,dim)
+ :comp+(long long)(item-wn)*dim;
+ float z=0;for(int d=lane;d<dim;d+=32)z+=qh[d]*v[d];
+ for(int n=16;n;n>>=1)z+=__shfl_down_sync(0xffffffff,z,n);
+ if(!lane)score[item]=z*scale;
+ }
+ __syncthreads();
+ if(!threadIdx.x){float m=-INFINITY;for(int i=0;i<total;i++)m=fmaxf(m,score[i]);
+ float d0=expf(sink[h]-m);
+ for(int i=0;i<total;i++){float a=expf(score[i]-m);d0+=a;score[i]=__bfloat162float(__float2bfloat16(a));}
+ den=d0;}
+ __syncthreads();
+ for(int d=threadIdx.x;d<dim;d+=blockDim.x){
+ float sum=0;
+ for(int i=0;i<wn;i++)sum+=score[i]*cached_row(win,chunk,abs_base+wo+i,chunk_start,window,dim)[d];
+ for(int i=0;i<cn;i++)sum+=score[wn+i]*comp[(long long)i*dim+d];
+ out[((long long)t*heads+h)*dim+d]=__bfloat162float(__float2bfloat16(sum/den));
+ }
+}
+
+extern "C" int dsv4_cuda_sparse_attn_batch_cached(int device,int layer,const float*q,
+ const float*chunk,int chunk_start,const float*sinks,const int*meta,int abs_base,
+ int comp_limit,int heads,int dim,int tokens,float scale,float*out){
+ Dev*c=ctx(device);
+ if(!c||layer<0||layer>=DSV4_KV_CACHE_LAYERS||!g_kv_ring[layer]||!q||!chunk||!sinks||
+ !meta||!out||abs_base<0||chunk_start<0||heads<1||dim<1||tokens<1||!(scale>0.f))return 0;
+ if(comp_limit>0&&(!g_kv_comp[layer]||g_kv_comp_cap[layer]<comp_limit))return 0;
+ int window=g_kv_ring_rows[layer],max_total=0;
+ for(int t=0;t<tokens;t++){
+ int wn=meta[3*t+1],cn=meta[3*t+2],tt=wn+cn;
+ if(meta[3*t]<0||wn<1||wn>window||cn<0||cn>comp_limit)return 0;
+ if(abs_base+meta[3*t]+wn>chunk_start+tokens)return 0;
+ if(tt>max_total)max_total=tt;
+ }
+ size_t qb=(size_t)tokens*heads*dim*4,cb=(size_t)tokens*dim*4,sb=(size_t)heads*4,
+ mb=(size_t)tokens*3*sizeof(int),sm=(size_t)max_total*4;
+ if(sm>96*1024)return 0;
+ if(!ok(cudaSetDevice(device),"select cached sparse attention device")||
+ !buf((void**)&c->p2,&c->p2cap,qb)||!buf((void**)&c->p4,&c->p4cap,qb)||
+ !buf((void**)&c->p3,&c->p3cap,cb)||
+ !buf((void**)&c->aux1,&c->aux1cap,sb)||!buf((void**)&c->aux2,&c->aux2cap,mb))return 0;
+ if(!ok(cudaMemcpyAsync(c->p2,q,qb,cudaMemcpyHostToDevice,c->stream),"cached attention query upload")||
+ !ok(cudaMemcpyAsync(c->p3,chunk,cb,cudaMemcpyHostToDevice,c->stream),"cached attention chunk upload")||
+ !ok(cudaMemcpyAsync(c->aux1,sinks,sb,cudaMemcpyHostToDevice,c->stream),"cached attention sink upload")||
+ !ok(cudaMemcpyAsync(c->aux2,meta,mb,cudaMemcpyHostToDevice,c->stream),"cached attention meta upload"))return 0;
+ sparse_attn_batch_cached_kernel<<<dim3(heads,tokens),256,sm,c->stream>>>(
+ c->p4,c->p2,g_kv_ring[layer],c->p3,g_kv_comp[layer],c->aux1,(int*)c->aux2,
+ abs_base,chunk_start,window,heads,dim,scale);
+ return ok(cudaGetLastError(),"cached sparse attention launch")&&
+ ok(cudaMemcpyAsync(out,c->p4,qb,cudaMemcpyDeviceToHost,c->stream),"cached attention download")&&
+ ok(cudaStreamSynchronize(c->stream),"cached sparse attention sync");
+}
+
+/* Indexed variant: past index_topk the indexer selects a strict subset of the
+ * compressed prefix, so meta[3t+2] becomes the SELECTED count and sel holds
+ * each token's compressed ordinals (score order — the same order the CPU
+ * reference accumulates, keeping fp32 sums bitwise identical). */
+__global__ void sparse_attn_batch_cached_idx_kernel(float*out,const float*q,const float*win,
+ const float*chunk,const float*comp,const float*sink,const int*meta,const int*sel,
+ int selstride,int abs_base,int chunk_start,int window,int heads,int dim,float scale){
+ int t=blockIdx.y,h=blockIdx.x;
+ int wo=meta[3*t],wn=meta[3*t+1],cn=meta[3*t+2],total=wn+cn;
+ const int*ts=sel+(long long)t*selstride;
+ const float*qh=q+((long long)t*heads+h)*dim;
+ extern __shared__ float score[];
+ __shared__ float den;
+ int warp=threadIdx.x>>5,lane=threadIdx.x&31;
+ for(int item=warp;item<total;item+=blockDim.x/32){
+ const float*v=item<wn?cached_row(win,chunk,abs_base+wo+item,chunk_start,window,dim)
+ :comp+(long long)ts[item-wn]*dim;
+ float z=0;for(int d=lane;d<dim;d+=32)z+=qh[d]*v[d];
+ for(int n=16;n;n>>=1)z+=__shfl_down_sync(0xffffffff,z,n);
+ if(!lane)score[item]=z*scale;
+ }
+ __syncthreads();
+ if(!threadIdx.x){float m=-INFINITY;for(int i=0;i<total;i++)m=fmaxf(m,score[i]);
+ float d0=expf(sink[h]-m);
+ for(int i=0;i<total;i++){float a=expf(score[i]-m);d0+=a;score[i]=__bfloat162float(__float2bfloat16(a));}
+ den=d0;}
+ __syncthreads();
+ for(int d=threadIdx.x;d<dim;d+=blockDim.x){
+ float sum=0;
+ for(int i=0;i<wn;i++)sum+=score[i]*cached_row(win,chunk,abs_base+wo+i,chunk_start,window,dim)[d];
+ for(int i=0;i<cn;i++)sum+=score[wn+i]*comp[(long long)ts[i]*dim+d];
+ out[((long long)t*heads+h)*dim+d]=__bfloat162float(__float2bfloat16(sum/den));
+ }
+}
+
+extern "C" int dsv4_cuda_sparse_attn_batch_cached_idx(int device,int layer,const float*q,
+ const float*chunk,int chunk_start,const float*sinks,const int*meta,const int*sel,
+ int selstride,int abs_base,int comp_limit,int heads,int dim,int tokens,
+ float scale,float*out){
+ Dev*c=ctx(device);
+ if(!c||layer<0||layer>=DSV4_KV_CACHE_LAYERS||!g_kv_ring[layer]||!q||!chunk||!sinks||
+ !meta||!sel||selstride<1||!out||abs_base<0||chunk_start<0||heads<1||dim<1||
+ tokens<1||!(scale>0.f))return 0;
+ if(comp_limit>0&&(!g_kv_comp[layer]||g_kv_comp_cap[layer]<comp_limit))return 0;
+ int window=g_kv_ring_rows[layer],max_total=0;
+ for(int t=0;t<tokens;t++){
+ int wn=meta[3*t+1],cn=meta[3*t+2],tt=wn+cn;
+ if(meta[3*t]<0||wn<1||wn>window||cn<0||cn>selstride||cn>comp_limit)return 0;
+ if(abs_base+meta[3*t]+wn>chunk_start+tokens)return 0;
+ for(int i=0;i<cn;i++){int o=sel[(long long)t*selstride+i];if(o<0||o>=comp_limit)return 0;}
+ if(tt>max_total)max_total=tt;
+ }
+ size_t qb=(size_t)tokens*heads*dim*4,cb=(size_t)tokens*dim*4,sb=(size_t)heads*4,
+ mb=(size_t)tokens*3*sizeof(int),ib=(size_t)tokens*selstride*sizeof(int),
+ sm=(size_t)max_total*4;
+ if(sm>96*1024)return 0;
+ if(!ok(cudaSetDevice(device),"select cached idx sparse attention device")||
+ !buf((void**)&c->p2,&c->p2cap,qb)||!buf((void**)&c->p4,&c->p4cap,qb)||
+ !buf((void**)&c->p3,&c->p3cap,cb)||
+ !buf((void**)&c->aux1,&c->aux1cap,sb)||!buf((void**)&c->aux2,&c->aux2cap,mb)||
+ !buf((void**)&c->aux3,&c->aux3cap,ib))return 0;
+ if(!ok(cudaMemcpyAsync(c->p2,q,qb,cudaMemcpyHostToDevice,c->stream),"cached idx attention query upload")||
+ !ok(cudaMemcpyAsync(c->p3,chunk,cb,cudaMemcpyHostToDevice,c->stream),"cached idx attention chunk upload")||
+ !ok(cudaMemcpyAsync(c->aux1,sinks,sb,cudaMemcpyHostToDevice,c->stream),"cached idx attention sink upload")||
+ !ok(cudaMemcpyAsync(c->aux2,meta,mb,cudaMemcpyHostToDevice,c->stream),"cached idx attention meta upload")||
+ !ok(cudaMemcpyAsync(c->aux3,sel,ib,cudaMemcpyHostToDevice,c->stream),"cached idx attention index upload"))return 0;
+ sparse_attn_batch_cached_idx_kernel<<<dim3(heads,tokens),256,sm,c->stream>>>(
+ c->p4,c->p2,g_kv_ring[layer],c->p3,g_kv_comp[layer],c->aux1,(int*)c->aux2,(int*)c->aux3,
+ selstride,abs_base,chunk_start,window,heads,dim,scale);
+ return ok(cudaGetLastError(),"cached idx sparse attention launch")&&
+ ok(cudaMemcpyAsync(out,c->p4,qb,cudaMemcpyDeviceToHost,c->stream),"cached idx attention download")&&
+ ok(cudaStreamSynchronize(c->stream),"cached idx sparse attention sync");
+}
+
+/* Batched indexer scoring for prefill. One thread per (token, candidate):
+ * score = sum_h relu(q_h . k) * w_h with the SAME sequential fp32 order as the
+ * CPU reference (dims inner, heads outer, separately rounded mul/add) so the
+ * host-side top-k sort sees bitwise-identical scores. queries [tokens][heads*dim]
+ * (already hadamard/fp4-rounded), keys [count][dim], head_w [tokens][heads],
+ * counts[t] = candidates visible to token t. Candidates >= counts[t] get 0. */
+__global__ void indexer_score_kernel(float*scores,const float*queries,const float*keys,
+ const float*head_w,const int*counts,int heads,int dim,int count){
+ int t=blockIdx.y,c=blockIdx.x*blockDim.x+threadIdx.x;
+ extern __shared__ float qs[];
+ int qn=heads*dim;
+ for(int i=threadIdx.x;i<qn;i+=blockDim.x)qs[i]=queries[(long long)t*qn+i];
+ __syncthreads();
+ if(c>=count)return;
+ float score=0.f;
+ if(c<counts[t]){
+ const float*k=keys+(long long)c*dim;
+ for(int h=0;h<heads;h++){
+ const float*q=qs+h*dim;
+ float dot=0.f;
+ /* The CPU build (MinGW GCC, -O3 -march=native) does NOT contract
+ * these into FMAs — measured — so keep mul and add separately
+ * rounded here too. */
+ for(int i=0;i<dim;i++)dot=__fadd_rn(dot,__fmul_rn(q[i],k[i]));
+ score=__fadd_rn(score,__fmul_rn(fmaxf(dot,0.f),head_w[(long long)t*heads+h]));
+ }
+ }
+ scores[(long long)t*count+c]=score;
+}
+
+extern "C" int dsv4_cuda_indexer_score_batch(int device,const float*queries,const float*keys,
+ const float*head_w,const int*counts,int tokens,int heads,int dim,int count,
+ float*scores){
+ Dev*c=ctx(device);
+ if(!c||!queries||!keys||!head_w||!counts||!scores||tokens<1||heads<1||dim<1||count<1)return 0;
+ size_t qb=(size_t)tokens*heads*dim*4,kb=(size_t)count*dim*4,wb=(size_t)tokens*heads*4,
+ cb=(size_t)tokens*sizeof(int),sb=(size_t)tokens*count*4,sm=(size_t)heads*dim*4;
+ if(sm>96*1024)return 0;
+ if(!ok(cudaSetDevice(device),"select indexer score device")||
+ !buf((void**)&c->p2,&c->p2cap,qb)||!buf((void**)&c->p3,&c->p3cap,kb)||
+ !buf((void**)&c->p4,&c->p4cap,sb)||!buf((void**)&c->aux1,&c->aux1cap,wb)||
+ !buf((void**)&c->aux2,&c->aux2cap,cb))return 0;
+ if(!ok(cudaMemcpyAsync(c->p2,queries,qb,cudaMemcpyHostToDevice,c->stream),"indexer query upload")||
+ !ok(cudaMemcpyAsync(c->p3,keys,kb,cudaMemcpyHostToDevice,c->stream),"indexer key upload")||
+ !ok(cudaMemcpyAsync(c->aux1,head_w,wb,cudaMemcpyHostToDevice,c->stream),"indexer head weight upload")||
+ !ok(cudaMemcpyAsync(c->aux2,counts,cb,cudaMemcpyHostToDevice,c->stream),"indexer count upload"))return 0;
+ if(sm>48*1024&&cudaFuncSetAttribute(indexer_score_kernel,cudaFuncAttributeMaxDynamicSharedMemorySize,(int)sm)!=cudaSuccess)return 0;
+ indexer_score_kernel<<<dim3((count+127)/128,tokens),128,sm,c->stream>>>(
+ c->p4,c->p2,c->p3,c->aux1,(int*)c->aux2,heads,dim,count);
+ return ok(cudaGetLastError(),"indexer score launch")&&
+ ok(cudaMemcpyAsync(scores,c->p4,sb,cudaMemcpyDeviceToHost,c->stream),"indexer score download")&&
+ ok(cudaStreamSynchronize(c->stream),"indexer score sync");
+}
+
+/* Exact replica of the CPU reference matmul_fp8 (quant.h): per output row,
+ * per 128-column block a sequential fp32 accumulate of e4m3(w)*x, then a
+ * DOUBLE accumulate of block*scale across blocks, cast to float. Separately
+ * rounded mul/add (the CPU build does not contract to FMA), so results are
+ * bitwise the reference. x is the fp8-qdq'd activation (done on the host).
+ * One thread per (row, token); tokens are the fast dimension so the 64
+ * threads of a token block broadcast-read the same weight bytes. */
+__device__ __forceinline__ float e4m3_dev(uint8_t b){
+ int s=b>>7,e=(b>>3)&15,m=b&7;
+ float v;
+ if(e==15&&m==7)v=__int_as_float(0x7fc00000); /* nan */
+ else if(e==0)v=ldexpf((float)m/8.f,-6);
+ else v=ldexpf(1.f+(float)m/8.f,e-7);
+ return s?-v:v;
+}
+__global__ void fp8_ref_matmul_kernel(float*y,const uint8_t*w,const float*bscale,
+ const float*x,int rows,int cols,int tokens){
+ int t=threadIdx.x,o=blockIdx.x;
+ if(t>=tokens||o>=rows)return;
+ const uint8_t*wr=w+(long long)o*cols;
+ const float*xs=x+(long long)t*cols;
+ int nblk=(cols+127)/128;
+ const float*scl=bscale+(long long)(o/128)*nblk;
+ double a=0.0;
+ for(int bi=0;bi*128<cols;bi++){
+ int base=bi*128,blen=cols-base<128?cols-base:128;
+ float acc=0.f;
+ for(int i=base;i<base+blen;i++)acc=__fadd_rn(acc,__fmul_rn(e4m3_dev(wr[i]),xs[i]));
+ a=__dadd_rn(a,__dmul_rn((double)acc,(double)scl[bi]));
+ }
+ y[(long long)t*rows+o]=__double2float_rn(a);
+}
+
+/* rows8-PACKED replica: the resident CPU layout stores 8 consecutive rows'
+ * bytes per column ((tile*cols+col)*8 + r) and the AVX2 reference kernel
+ * accumulates sum += (x*w)*scale in fp32 sequentially over ALL columns (no
+ * per-block double). Same order here, so bitwise the AVX2 path. */
+__global__ void fp8_ref_matmul_rows8_kernel(float*y,const uint8_t*w,const float*bscale,
+ const float*x,int rows,int cols,int tokens){
+ int t=threadIdx.x,o=blockIdx.x;
+ if(t>=tokens||o>=rows)return;
+ const float*xs=x+(long long)t*cols;
+ int nblk=cols/128;
+ const float*scl=bscale+(long long)(o/128)*nblk;
+ long long tile=o>>3;int r=o&7;
+ float sum=0.f;
+ for(int base=0;base<cols;base+=128){
+ float sc=scl[base/128];
+ for(int i=base;i<base+128;i++){
+ float v=e4m3_dev(w[((tile*cols)+i)*8+r]);
+ sum=__fadd_rn(sum,__fmul_rn(__fmul_rn(xs[i],v),sc));
+ }
+ }
+ y[(long long)t*rows+o]=sum;
+}
+
+extern "C" int dsv4_cuda_fp8_ref_matmul(int device,const uint8_t*w,const float*bscale,
+ int rows,int cols,int packed_rows8,const float*x,int tokens,float*y){
+ Dev*c=ctx(device);
+ if(!c||!w||!bscale||!x||!y||rows<1||cols<1||cols%128||tokens<1||tokens>1024||
+ (packed_rows8&&rows%8))return 0;
+ size_t wb=(size_t)rows*cols,sb=(size_t)((rows+127)/128)*(cols/128)*4,
+ xb=(size_t)tokens*cols*4,yb=(size_t)tokens*rows*4;
+ if(!ok(cudaSetDevice(device),"select fp8 ref matmul device")||
+ !buf((void**)&c->p1,&c->p1cap,wb)||!buf((void**)&c->aux1,&c->aux1cap,sb)||
+ !buf((void**)&c->p2,&c->p2cap,xb)||!buf((void**)&c->p4,&c->p4cap,yb))return 0;
+ if(!ok(cudaMemcpyAsync(c->p1,w,wb,cudaMemcpyHostToDevice,c->stream),"fp8 ref weight upload")||
+ !ok(cudaMemcpyAsync(c->aux1,bscale,sb,cudaMemcpyHostToDevice,c->stream),"fp8 ref scale upload")||
+ !ok(cudaMemcpyAsync(c->p2,x,xb,cudaMemcpyHostToDevice,c->stream),"fp8 ref activation upload"))return 0;
+ int threads=tokens<32?32:((tokens+31)/32)*32;
+ if(packed_rows8)
+ fp8_ref_matmul_rows8_kernel<<<rows,threads,0,c->stream>>>(c->p4,(const uint8_t*)c->p1,c->aux1,c->p2,rows,cols,tokens);
+ else
+ fp8_ref_matmul_kernel<<<rows,threads,0,c->stream>>>(c->p4,(const uint8_t*)c->p1,c->aux1,c->p2,rows,cols,tokens);
+ return ok(cudaGetLastError(),"fp8 ref matmul launch")&&
+ ok(cudaMemcpyAsync(y,c->p4,yb,cudaMemcpyDeviceToHost,c->stream),"fp8 ref matmul download")&&
+ ok(cudaStreamSynchronize(c->stream),"fp8 ref matmul sync");
+}
+
+__global__ void sparse_attn_batch_kernel(float*out,const float*q,const float*vals,const float*sink,
+ const int*meta,int comp_base,int heads,int dim,float scale){
+ int t=blockIdx.y,h=blockIdx.x;
+ int wo=meta[3*t],wn=meta[3*t+1],cn=meta[3*t+2],total=wn+cn;
+ const float*qh=q+((long long)t*heads+h)*dim;
+ extern __shared__ float score[];
+ __shared__ float den;
+ int warp=threadIdx.x>>5,lane=threadIdx.x&31;
+ for(int item=warp;item<total;item+=blockDim.x/32){
+ const float*v=vals+(long long)(item<wn?wo+item:comp_base+item-wn)*dim;
+ float z=0;for(int d=lane;d<dim;d+=32)z+=qh[d]*v[d];
+ for(int n=16;n;n>>=1)z+=__shfl_down_sync(0xffffffff,z,n);
+ if(!lane)score[item]=z*scale;
+ }
+ __syncthreads();
+ if(!threadIdx.x){float m=-INFINITY;for(int i=0;i<total;i++)m=fmaxf(m,score[i]);
+ float d0=expf(sink[h]-m);
+ for(int i=0;i<total;i++){float a=expf(score[i]-m);d0+=a;score[i]=__bfloat162float(__float2bfloat16(a));}
+ den=d0;}
+ __syncthreads();
+ for(int d=threadIdx.x;d<dim;d+=blockDim.x){
+ float sum=0;
+ for(int i=0;i<wn;i++)sum+=score[i]*vals[(long long)(wo+i)*dim+d];
+ for(int i=0;i<cn;i++)sum+=score[wn+i]*vals[(long long)(comp_base+i)*dim+d];
+ out[((long long)t*heads+h)*dim+d]=__bfloat162float(__float2bfloat16(sum/den));
+ }
+}
+extern "C" int dsv4_cuda_sparse_attn_batch(int device,const float*q,const float*vals,const float*sinks,
+ const int*meta,int value_rows,int comp_base,int heads,int dim,int tokens,float scale,float*out){
+ Dev*c=ctx(device);
+ if(!c||!q||!vals||!sinks||!meta||!out||value_rows<1||comp_base<0||comp_base>value_rows||
+ heads<1||dim<1||tokens<1||!(scale>0.f))return 0;
+ int max_total=0;
+ for(int t=0;t<tokens;t++){
+ int wn=meta[3*t+1],cn=meta[3*t+2],tt=wn+cn;
+ if(meta[3*t]<0||wn<1||cn<0||meta[3*t]+wn>comp_base||comp_base+cn>value_rows)return 0;
+ if(tt>max_total)max_total=tt;
+ }
+ size_t qb=(size_t)tokens*heads*dim*4,vb=(size_t)value_rows*dim*4,sb=(size_t)heads*4,
+ mb=(size_t)tokens*3*sizeof(int),sm=(size_t)max_total*4;
+ if(sm>96*1024)return 0;
+ if(!ok(cudaSetDevice(device),"select batched sparse attention device")||
+ !buf((void**)&c->p2,&c->p2cap,qb)||!buf((void**)&c->p1,&c->p1cap,vb)||
+ !buf((void**)&c->p4,&c->p4cap,qb)||!buf((void**)&c->aux1,&c->aux1cap,sb)||
+ !buf((void**)&c->aux2,&c->aux2cap,mb))return 0;
+ if(!ok(cudaMemcpyAsync(c->p2,q,qb,cudaMemcpyHostToDevice,c->stream),"batched attention query upload")||
+ !ok(cudaMemcpyAsync(c->p1,vals,vb,cudaMemcpyHostToDevice,c->stream),"batched attention KV upload")||
+ !ok(cudaMemcpyAsync(c->aux1,sinks,sb,cudaMemcpyHostToDevice,c->stream),"batched attention sink upload")||
+ !ok(cudaMemcpyAsync(c->aux2,meta,mb,cudaMemcpyHostToDevice,c->stream),"batched attention meta upload"))return 0;
+ if(sm>48*1024&&!ok(cudaFuncSetAttribute(sparse_attn_batch_kernel,cudaFuncAttributeMaxDynamicSharedMemorySize,(int)sm),"batched sparse attention shared memory"))return 0;
+ sparse_attn_batch_kernel<<<dim3(heads,tokens),256,sm,c->stream>>>(c->p4,c->p2,c->p1,c->aux1,(int*)c->aux2,comp_base,heads,dim,scale);
+ return ok(cudaGetLastError(),"batched sparse attention launch")&&
+ ok(cudaMemcpyAsync(out,c->p4,qb,cudaMemcpyDeviceToHost,c->stream),"batched attention context download")&&
+ ok(cudaStreamSynchronize(c->stream),"batched sparse attention sync");
+}
+/* Prefill batch GEMM over a resident bf16 matrix (compressor / indexer
+ * projections). Same tiling as mm_batch; numerics match the CPU reference:
+ * bf16-decoded weights, fp32 accumulation, no activation quantization. */
+__global__ void mm_bf16_batch(const __nv_bfloat16*w,const float*x,float*y,int O,int I,int T){
+ int o=blockIdx.x;if(o>=O)return;int tbase=blockIdx.y*16,tcount=T-tbase<16?T-tbase:16;
+ float sum[16]={};
+ for(int i=threadIdx.x;i<I;i+=blockDim.x){float v=__bfloat162float(w[(long long)o*I+i]);
+ #pragma unroll
+ for(int t=0;t<16;t++)if(t<tcount)sum[t]+=x[(long long)(tbase+t)*I+i]*v;}
+ __shared__ float s[16][256];
+ #pragma unroll
+ for(int t=0;t<16;t++)s[t][threadIdx.x]=sum[t];
+ __syncthreads();
+ for(int n=128;n;n>>=1){if(threadIdx.x<n){
+ #pragma unroll
+ for(int t=0;t<16;t++)s[t][threadIdx.x]+=s[t][threadIdx.x+n];}__syncthreads();}
+ if(!threadIdx.x)for(int t=0;t<tcount;t++)y[(long long)(tbase+t)*O+o]=s[t][0];
+}
+extern "C" int dsv4_cuda_matmul_bf16_batch(Dsv4CudaTensor*t,const float*x,int tokens,float*y){
+ Dev*c=t?ctx(t->device):nullptr;
+ if(!c||!x||!y||tokens<1||t->fmt!=16||!ok(cudaSetDevice(t->device),"select bf16 batch device"))return 0;
+ size_t xb=(size_t)tokens*t->I*4,yb=(size_t)tokens*t->O*4;
+ if(!buf((void**)&c->dx,&c->xcap,xb)||!buf((void**)&c->dy,&c->ycap,yb))return 0;
+ if(!ok(cudaMemcpyAsync(c->dx,x,xb,cudaMemcpyHostToDevice,c->stream),"bf16 batch upload"))return 0;
+ mm_bf16_batch<<<dim3(t->O,(tokens+15)/16),256,0,c->stream>>>((__nv_bfloat16*)t->w,c->dx,c->dy,t->O,t->I,tokens);
+ return ok(cudaGetLastError(),"bf16 batch launch")&&
+ ok(cudaMemcpyAsync(y,c->dy,yb,cudaMemcpyDeviceToHost,c->stream),"bf16 batch download")&&
+ ok(cudaStreamSynchronize(c->stream),"bf16 batch sync");
+}
__device__ __forceinline__ int decode_pos(const int *state){return state[1];}
__device__ __forceinline__ int compression_ready(const int *state,int ratio){return ratio&&decode_pos(state)%ratio==ratio-1;}
__global__ void rope_heads_dynamic(float*x,int heads,int dim,int rope,const int*state,int delta,int inverse,int predicate,const float*cs,const float*sn){if(predicate&&!compression_ready(state,predicate))return;int pos=decode_pos(state)+delta,p=blockIdx.x*blockDim.x+threadIdx.x,pairs=rope/2;if(p<heads*pairs){int h=p/pairs,k=p%pairs,off=h*dim+dim-rope+2*k;float s=inverse?-sn[(long long)pos*pairs+k]:sn[(long long)pos*pairs+k],u=x[off],v=x[off+1];x[off]=__bfloat162float(__float2bfloat16(u*cs[(long long)pos*pairs+k]-v*s));x[off+1]=__bfloat162float(__float2bfloat16(u*s+v*cs[(long long)pos*pairs+k]));}}
__global__ void cache_store_dynamic(float*cache,const float*kv,const int*state,int window,int dim){int d=blockIdx.x*blockDim.x+threadIdx.x;if(d<dim)cache[(long long)(decode_pos(state)%window)*dim+d]=kv[d];}
+/* Unconditional, not FLASHINFER-gated: the DeepGEMM batched wo path
+ * (dsv4_cuda_wo) launches these too, and a DEEPGEMM=1 FLASHINFER=0 build
+ * otherwise fails to compile. */
__global__ void gather_group_rows(float*out,const float*in,int tokens,int groups,int width,int group){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)tokens*width;if(p<n){int row=p/width,col=p%width;out[p]=in[((long long)row*groups+group)*width+col];}}
__global__ void scatter_group_rows(float*out,const float*in,int tokens,int groups,int width,int group){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)tokens*width;if(p<n){int row=p/width,col=p%width;out[((long long)row*groups+group)*width+col]=in[p];}}
__global__ void compress_store_dynamic(float*values,float*scores,const float*v,const float*g,const float*ape,const int*state,int ratio,int overlap,int O){int d=blockIdx.x*blockDim.x+threadIdx.x;if(d<O){int phase=decode_pos(state)%ratio,slot=(overlap?ratio:0)+phase;values[(long long)slot*O+d]=v[d];scores[(long long)slot*O+d]=g[d]+ape[(long long)phase*O+d];}}
@@ -641,7 +1194,7 @@ __global__ void final_mhc_coeff(float*pre,const float*res,const float*fn,const f
for(int k=128;k;k>>=1){if(threadIdx.x<k)for(int m=0;m<=M;m++)sum[m][threadIdx.x]+=sum[m][threadIdx.x+k];__syncthreads();}if(threadIdx.x<M){float inv=rsqrtf(sum[0][0]/MH+eps);pre[threadIdx.x]=1.f/(1.f+expf(-(sum[threadIdx.x+1][0]*inv*scale[0]+base[threadIdx.x])))+pre_eps;}}
__global__ void final_mhc_mix(float*out,const float*res,const float*pre,int M,int H){int h=blockIdx.x*blockDim.x+threadIdx.x;if(h<H){float v=0;for(int m=0;m<M;m++)v+=pre[m]*res[(long long)m*H+h];out[h]=__bfloat162float(__float2bfloat16(v));}}
-extern "C" int dsv4_cuda_init(const int *dev,int n){if(n<1||n>16)return 0;float scale[256];for(int b=0;b<256;b++)scale[b]=b?ldexpf(1.f,b-127):ldexpf(1.f,-127);ng=0;for(int i=0;i<n;i++){cudaDeviceProp p{};if(!ok(cudaSetDevice(dev[i]),"select device")||!ok(cudaMemcpyToSymbol(e8_table,scale,sizeof(scale)),"scale table upload")||!ok(cudaGetDeviceProperties(&p,dev[i]),"device properties")||!ok(cudaStreamCreateWithFlags(&g[i].stream,cudaStreamNonBlocking),"stream")||!ok(cudaStreamCreateWithFlags(&g[i].aux,cudaStreamNonBlocking),"aux stream")||!ok(cudaEventCreateWithFlags(&g[i].ready,cudaEventDisableTiming),"device event")||!ok(cudaEventCreateWithFlags(&g[i].fork,cudaEventDisableTiming),"fork event")||!ok(cudaEventCreateWithFlags(&g[i].join,cudaEventDisableTiming),"join event")||!bok(cublasLtCreate(&g[i].lt),"cuBLASLt create")||!ok(cudaMalloc(&g[i].tc_workspace,32<<20),"Tensor Core workspace")||!ok(cudaMalloc(&g[i].decode_state,2*sizeof(int)),"decode state allocation"))return 0;g[i].id=dev[i];ng++;fprintf(stderr,"[DSV4 CUDA] device %d: %s %.1f GB sm_%d%d\n",dev[i],p.name,p.totalGlobalMem/1e9,p.major,p.minor);}for(int i=0;i<n;i++)for(int j=0;j<n;j++)if(i!=j){int can=0;if(!ok(cudaDeviceCanAccessPeer(&can,dev[i],dev[j]),"peer query"))return 0;if(can){if(!ok(cudaSetDevice(dev[i]),"select peer device"))return 0;cudaError_t e=cudaDeviceEnablePeerAccess(dev[j],0);if(e!=cudaSuccess&&e!=cudaErrorPeerAccessAlreadyEnabled)return ok(e,"enable peer access");}}
+extern "C" int dsv4_cuda_init(const int *dev,int n){if(n<1||n>16)return 0;float scale[256];for(int b=0;b<256;b++)scale[b]=b?ldexpf(1.f,b-127):ldexpf(1.f,-127);ng=0;for(int i=0;i<n;i++){cudaDeviceProp p{};if(!ok(cudaSetDevice(dev[i]),"select device")||!ok(cudaMemcpyToSymbol(e8_table,scale,sizeof(scale)),"scale table upload")||!ok(cudaGetDeviceProperties(&p,dev[i]),"device properties")||!ok(cudaStreamCreateWithFlags(&g[i].stream,cudaStreamNonBlocking),"stream")||!ok(cudaStreamCreateWithFlags(&g[i].aux,cudaStreamNonBlocking),"aux stream")||!ok(cudaEventCreateWithFlags(&g[i].ready,cudaEventDisableTiming),"device event")||!ok(cudaEventCreateWithFlags(&g[i].fork,cudaEventDisableTiming),"fork event")||!ok(cudaEventCreateWithFlags(&g[i].join,cudaEventDisableTiming),"join event")||!bok(cublasLtCreate(&g[i].lt),"cuBLASLt create")||!ok(cudaMalloc(&g[i].tc_workspace,32<<20),"Tensor Core workspace")||!ok(cudaMalloc(&g[i].decode_state,2*sizeof(int)),"decode state allocation"))return 0;g[i].id=dev[i];g[i].sms=p.multiProcessorCount;ng++;fprintf(stderr,"[DSV4 CUDA] device %d: %s %.1f GB sm_%d%d\n",dev[i],p.name,p.totalGlobalMem/1e9,p.major,p.minor);}for(int i=0;i<n;i++)for(int j=0;j<n;j++)if(i!=j){int can=0;if(!ok(cudaDeviceCanAccessPeer(&can,dev[i],dev[j]),"peer query"))return 0;if(can){if(!ok(cudaSetDevice(dev[i]),"select peer device"))return 0;cudaError_t e=cudaDeviceEnablePeerAccess(dev[j],0);if(e!=cudaSuccess&&e!=cudaErrorPeerAccessAlreadyEnabled)return ok(e,"enable peer access");}}
#ifdef COLI_DSV4_NCCL
if(n==6)for(int pair=0;pair<3;pair++){int ids[2]={dev[2*pair],dev[2*pair+1]};ncclComm_t comm[2];if(!nok(ncclCommInitAll(comm,2,ids),"EP2 communicator init"))return 0;g[2*pair].ep_comm=comm[0];g[2*pair+1].ep_comm=comm[1];}
#endif
@@ -655,7 +1208,7 @@ extern "C" void dsv4_cuda_shutdown(void){
}
for(int i=0;i<ng;i++){cudaSetDevice(g[i].id);if(g[i].dx)cudaFree(g[i].dx);if(g[i].dy)cudaFree(g[i].dy);if(g[i].p1)cudaFree(g[i].p1);if(g[i].p2)cudaFree(g[i].p2);if(g[i].p3)cudaFree(g[i].p3);if(g[i].p4)cudaFree(g[i].p4);if(g[i].aux1)cudaFree(g[i].aux1);if(g[i].aux2)cudaFree(g[i].aux2);if(g[i].aux3)cudaFree(g[i].aux3);if(g[i].mvdesc)cudaFree(g[i].mvdesc);if(g[i].expert_weights)cudaFree(g[i].expert_weights);if(g[i].expert_ids)cudaFree(g[i].expert_ids);if(g[i].tcw)cudaFree(g[i].tcw);if(g[i].tcs)cudaFree(g[i].tcs);if(g[i].tcx)cudaFree(g[i].tcx);if(g[i].tcxs)cudaFree(g[i].tcxs);if(g[i].tc_workspace)cudaFree(g[i].tc_workspace);
#ifdef COLI_DSV4_DEEPGEMM
- if(g[i].dga1)cudaFree(g[i].dga1);if(g[i].dga2)cudaFree(g[i].dga2);if(g[i].dgsfa1)cudaFree(g[i].dgsfa1);if(g[i].dgsfa2)cudaFree(g[i].dgsfa2);if(g[i].dglayout)cudaFree(g[i].dglayout);if(g[i].dgmm1)cudaFree(g[i].dgmm1);if(g[i].dgmm2)cudaFree(g[i].dgmm2);if(g[i].dgref)cudaFree(g[i].dgref);if(g[i].dgdensews)cudaFree(g[i].dgdensews);
+ if(g[i].dga1)cudaFree(g[i].dga1);if(g[i].dga2)cudaFree(g[i].dga2);if(g[i].dgsfa1)cudaFree(g[i].dgsfa1);if(g[i].dgsfa2)cudaFree(g[i].dgsfa2);if(g[i].dglayout)cudaFree(g[i].dglayout);if(g[i].dgrowmap)cudaFree(g[i].dgrowmap);if(g[i].dgmm1)cudaFree(g[i].dgmm1);if(g[i].dgmm2)cudaFree(g[i].dgmm2);if(g[i].dgref)cudaFree(g[i].dgref);if(g[i].dgdensews)cudaFree(g[i].dgdensews);
#endif
for(int p=0;p<2;p++)plan_free(&g[i].plans[p]);if(g[i].lt)cublasLtDestroy(g[i].lt);if(g[i].hx)cudaFreeHost(g[i].hx);if(g[i].hy)cudaFreeHost(g[i].hy);if(g[i].ready)cudaEventDestroy(g[i].ready);if(g[i].fork)cudaEventDestroy(g[i].fork);if(g[i].join)cudaEventDestroy(g[i].join);cudaStreamDestroy(g[i].aux);cudaStreamDestroy(g[i].stream);}ng=0;}
extern "C" Dsv4CudaActivation *dsv4_cuda_activation_create(int device,long long elements){Dev*c=ctx(device);if(!c||elements<1||!ok(cudaSetDevice(device),"select activation device"))return nullptr;Dsv4CudaActivation*a=(Dsv4CudaActivation*)calloc(1,sizeof(*a));if(!a)return nullptr;a->device=device;a->elements=elements;if(!ok(cudaMalloc(&a->data,(size_t)elements*sizeof(float)),"activation allocation")){free(a);return nullptr;}return a;}
@@ -703,6 +1256,25 @@ extern "C" int dsv4_cuda_upload_fp8(Dsv4CudaTensor **t,const uint8_t*w,const uin
extern "C" int dsv4_cuda_upload_fp8_bf16(Dsv4CudaTensor **t,const uint8_t*w,const uint8_t*s,int O,int I,int d){return upload(t,w,(size_t)O*I,s,(size_t)((O+127)/128)*((I+127)/128),O,I,d,9);}
extern "C" int dsv4_cuda_upload_fp4(Dsv4CudaTensor **t,const uint8_t*w,const uint8_t*s,int O,int I,int d){if(!upload(t,w,(size_t)O*I/2,s,(size_t)O*I/32,O,I,d,4))return 0;
return 1;}
+/* In-place refill of an existing fp4 mirror with a same-shape expert: no
+ * cudaFree/cudaMalloc churn, DMA from page-locked slabs when possible. The
+ * copies are stream-ordered; sync=1 drains before returning (needed when the
+ * host slab may be released before the next stream sync). */
+static bool host_pinned(const void*p,size_t len);
+extern "C" int dsv4_cuda_tensor_refill_fp4(Dsv4CudaTensor *t,const uint8_t*w,const uint8_t*s,int O,int I,int sync){
+ if(!t||!w||!s||t->fmt!=4||t->O!=O||t->I!=I||!t->own_w||!t->own_scale||!t->w||!t->scale)return 0;
+ Dev*c=ctx(t->device);if(!c||!ok(cudaSetDevice(t->device),"select refill device"))return 0;
+ size_t wb=(size_t)O*I/2,sb=(size_t)O*I/32;
+ if(host_pinned(w,wb)&&host_pinned(s,sb)){
+ if(!ok(cudaMemcpyAsync(t->w,w,wb,cudaMemcpyHostToDevice,c->stream),"refill weight upload")||
+ !ok(cudaMemcpyAsync(t->scale,s,sb,cudaMemcpyHostToDevice,c->stream),"refill scale upload"))return 0;
+ if(sync&&!ok(cudaStreamSynchronize(c->stream),"refill drain"))return 0;
+ }else{
+ if(!ok(cudaMemcpy(t->w,w,wb,cudaMemcpyHostToDevice),"refill weight upload")||
+ !ok(cudaMemcpy(t->scale,s,sb,cudaMemcpyHostToDevice),"refill scale upload"))return 0;
+ }
+ return 1;
+}
extern "C" int dsv4_cuda_upload_bf16(Dsv4CudaTensor **t,const uint16_t*w,int O,int I,int d){return upload(t,w,(size_t)O*I*2,nullptr,0,O,I,d,16);}
extern "C" int dsv4_cuda_upload_f32(Dsv4CudaTensor **t,const float*w,int O,int I,int d){return upload(t,w,(size_t)O*I*4,nullptr,0,O,I,d,32);}
extern "C" int dsv4_cuda_mhc_pre(const Dsv4CudaActivation*r,Dsv4CudaTensor*fn,Dsv4CudaTensor*scale,Dsv4CudaTensor*base,int M,int H,float rms_eps,float pre_eps,float sink_eps,float post_mult,int sink_iters,Dsv4CudaActivation*state,Dsv4CudaActivation*input){
@@ -894,6 +1466,10 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4C
if (qk_rope)
rope_heads_dynamic<<<(qk_rope / 2 + 255) / 256, 256, 0, comp>>>(
comp3, 1, dim, qk_rope, c->decode_state, 1 - ratio, 0, ratio, cache->comp_cos, cache->comp_sin);
+ /* The FlashInfer cache packer performs the model's UE8M0 FP8
+ * quantization. Quantizing and dequantizing here first would apply
+ * a second, differently-scaled FP8 round before the real cache
+ * store. Keep the simulation only for the float-cache fallback. */
if (dim > qk_rope)
fp8_sim64_dynamic<<<(dim - qk_rope + 63) / 64, 64, 0, comp>>>(comp3, dim - qk_rope,
c->decode_state, ratio);
@@ -919,8 +1495,9 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4C
{fp8_sim<<<(R + 127) / 128, 128, 0, c->stream>>>(c->p1, 1, R);
run_mv<8>((uint8_t *)qb->w, qb->scale, c->p1, c->p2, Q, R, 1, c->stream);}
bf16_round<<<(Q + 255) / 256, 256, 0, c->stream>>>(c->p2, Q);
- head_rmsnorm<<<heads, 256, 0, c->stream>>>(c->p2, heads, dim, eps);
- if (qk_rope)
+ int sparse=0;
+ if(!sparse)head_rmsnorm<<<heads, 256, 0, c->stream>>>(c->p2, heads, dim, eps);
+ if (qk_rope&&!sparse)
rope_heads_dynamic<<<(heads * (qk_rope / 2) + 255) / 256, 256, 0, c->stream>>>(
c->p2, heads, dim, qk_rope, c->decode_state, 0, 0, 0, cache->rope_cos, cache->rope_sin);
#ifdef COLI_DSV4_DEEPGEMM
@@ -929,11 +1506,10 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4C
run_mv<8>((uint8_t *)kv->w, kv->scale, c->dx, c->p3, K, H, 1, c->stream);
if (!fused_qkv) bf16_round<<<(K + 255) / 256, 256, 0, c->stream>>>(c->p3, K);
rmsnorm_f32<<<1, 256, 0, c->stream>>>(c->p3, fused_qkv ? c->p1 + R : c->p3, (float *)kvn->w, K, eps);
- if (qk_rope)
+ if (qk_rope&&!sparse)
rope_heads_dynamic<<<(qk_rope / 2 + 255) / 256, 256, 0, c->stream>>>(
c->p3, 1, dim, qk_rope, c->decode_state, 0, 0, 0, cache->rope_cos, cache->rope_sin);
- if (K > qk_rope) fp8_sim64<<<(K - qk_rope + 63) / 64, 64, 0, c->stream>>>(c->p3, K - qk_rope);
- cache_store_dynamic<<<(K + 255) / 256, 256, 0, c->stream>>>(cache->kv, c->p3, c->decode_state, cache->window, K);
+ if(!sparse){if (K > qk_rope) fp8_sim64<<<(K - qk_rope + 63) / 64, 64, 0, c->stream>>>(c->p3, K - qk_rope);cache_store_dynamic<<<(K + 255) / 256, 256, 0, c->stream>>>(cache->kv, c->p3, c->decode_state, cache->window, K);}
if (async && !ok(cudaStreamWaitEvent(c->stream, c->join, 0), "attention join wait")) return 0;
attention_window_dynamic<<<heads, 256, (size_t)(cache->window + cache->max_compressed) * 4, c->stream>>>(
c->p4, c->p2, cache->kv, cache->compressed, (float *)sink->w, heads, dim, cache->window, c->decode_state,
@@ -974,7 +1550,21 @@ extern "C" int dsv4_cuda_attention_output_batch(const Dsv4CudaActivation*context
scatter_group_rows<<<((long long)tokens*N+255)/256,256,0,c->stream>>>(c->p1,c->aux2,tokens,groups,N,gidx);}
return dg_dense_batch_dispatch(c,wb,c->p1,output->data,tokens);
#else
- (void)context;(void)wa;(void)wb;(void)groups;(void)tokens;(void)output;return 0;
+ /* GENERIC grouped wo_a (fmt 9 view per group) + wo_b via the portable
+ * batch matmul; same gather/scatter structure as the DeepGEMM path. */
+ Dev*c=context?ctx(context->device):nullptr;int K=wa?wa->I:0,N=wa&&groups?wa->O/groups:0,WR=wa?wa->O:0,H=wb?wb->O:0;
+ if(!c||!wa||!wb||!output||groups<1||tokens<1||wa->fmt!=9||wa->O%groups||N%128||K%128||wb->fmt!=8||wb->I!=WR||
+ wa->device!=context->device||wb->device!=context->device||output->device!=context->device||
+ context->elements<(long long)tokens*groups*K||output->elements<(long long)tokens*H)return 0;
+ size_t in=(size_t)tokens*K*sizeof(float),part=(size_t)tokens*N*sizeof(float),joined=(size_t)tokens*WR*sizeof(float);
+ if(!ok(cudaSetDevice(context->device),"select generic attention output device")||!buf((void**)&c->aux1,&c->aux1cap,in)||
+ !buf((void**)&c->aux2,&c->aux2cap,part)||!buf((void**)&c->p1,&c->p1cap,joined))return 0;
+ for(int gidx=0;gidx<groups;gidx++){gather_group_rows<<<((long long)tokens*K+255)/256,256,0,c->stream>>>(c->aux1,context->data,tokens,groups,K,gidx);
+ Dsv4CudaTensor view=*wa;view.O=N;view.w=(uint8_t*)wa->w+(long long)gidx*N*K;view.scale=wa->scale+(long long)gidx*(N/128)*(K/128);
+ run_mm_batch(&view,c->aux1,c->aux2,tokens,c->stream);
+ scatter_group_rows<<<((long long)tokens*N+255)/256,256,0,c->stream>>>(c->p1,c->aux2,tokens,groups,N,gidx);}
+ run_mm_batch(wb,c->p1,output->data,tokens,c->stream);
+ return ok(cudaGetLastError(),"generic attention output launch");
#endif
}
@@ -1028,12 +1618,14 @@ extern "C" int dsv4_cuda_matvec_grouped(Dsv4CudaTensor *t,float *y,const float*x
return ok(cudaGetLastError(),"matvec launch")&&ok(cudaMemcpyAsync(y,c->dy,yb,cudaMemcpyDeviceToHost,c->stream),"result download")&&ok(cudaStreamSynchronize(c->stream),"matvec sync");}
extern "C" int dsv4_cuda_matvec(Dsv4CudaTensor*t,float*y,const float*x){return dsv4_cuda_matvec_grouped(t,y,x,1);}
extern "C" int dsv4_cuda_matmul_batch(Dsv4CudaTensor*t,const Dsv4CudaActivation*input,int tokens,Dsv4CudaActivation*output){
-#ifdef COLI_DSV4_DEEPGEMM
Dev*c=t?ctx(t->device):nullptr;if(!c||!input||!output||tokens<1||input->device!=t->device||output->device!=t->device||
input->elements<(long long)tokens*t->I||output->elements<(long long)tokens*t->O||!ok(cudaSetDevice(t->device),"select batched dense device"))return 0;
+#ifdef COLI_DSV4_DEEPGEMM
return dg_dense_batch_dispatch(c,t,input->data,output->data,tokens);
#else
- (void)t;(void)input;(void)tokens;(void)output;return 0;
+ if(t->fmt!=8&&t->fmt!=9&&t->fmt!=4)return 0;
+ run_mm_batch(t,input->data,output->data,tokens,c->stream);
+ return ok(cudaGetLastError(),"batched dense launch");
#endif
}
extern "C" int dsv4_cuda_head_argmax(Dsv4CudaTensor*t,const float*x,int*id,float*value){
@@ -1046,9 +1638,9 @@ extern "C" int dsv4_cuda_head_argmax(Dsv4CudaTensor*t,const float*x,int*id,float
if(!dg_map(&a,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dga1,I,1,(unsigned long long)I*2,64,64,CU_TENSOR_MAP_SWIZZLE_128B)||
!dg_map(&b,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,t->w,I,O,(unsigned long long)I*2,64,128,CU_TENSOR_MAP_SWIZZLE_128B)||
!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,O,1,(unsigned long long)O*2,64,64,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto gemm=deep_gemm::sm120_bf16_gemm_impl<0,O,I,1,64,128,64,128,128,128,3,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,true,2>;
+ auto gemm=c->sms>=170?deep_gemm::sm120_bf16_gemm_impl<0,O,I,1,64,128,64,128,128,128,3,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,true,2>:deep_gemm::sm120_bf16_gemm_impl<0,O,I,1,64,128,64,128,128,128,3,128,256,84,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,true,2>;
if(!ok(cudaFuncSetAttribute(gemm,cudaFuncAttributeMaxDynamicSharedMemorySize,90368),"output head DeepGEMM shared memory"))return 0;
- gemm<<<170,384,90368,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(DgOutput*)c->dga1,(DgOutput*)t->w,nullptr,nullptr,1,O,I,a,b,d);
+ gemm<<<(c->sms>=170?170:84),384,90368,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(DgOutput*)c->dga1,(DgOutput*)t->w,nullptr,nullptr,1,O,I,a,b,d);
dg_argmax_bf16<<<1,256,0,c->stream>>>(c->dgmm1,O,c->expert_ids,c->expert_weights);
return ok(cudaGetLastError(),"output head DeepGEMM launch")&&ok(cudaMemcpyAsync(id,c->expert_ids,sizeof(*id),cudaMemcpyDeviceToHost,c->stream),"output id download")&&
ok(cudaMemcpyAsync(value,c->expert_weights,sizeof(*value),cudaMemcpyDeviceToHost,c->stream),"output logit download")&&ok(cudaStreamSynchronize(c->stream),"output head sync");
@@ -1064,13 +1656,14 @@ extern "C" int dsv4_cuda_final_argmax(const Dsv4CudaActivation*r,Dsv4CudaTensor*
!buf((void**)&c->expert_ids,&c->expert_idcap,sizeof(int))||!buf((void**)&c->expert_weights,&c->expert_weightscap,sizeof(float)))return 0;
final_mhc_coeff<<<1,256,0,c->stream>>>(c->p1,r->data,(float*)fn->w,(float*)scale->w,(float*)base->w,M,H,eps,pre_eps);final_mhc_mix<<<(H+255)/256,256,0,c->stream>>>(c->dx,r->data,c->p1,M,H);rmsnorm_f32<<<1,256,0,c->stream>>>(c->dx,c->dx,(float*)norm->w,H,eps);dg_float_to_bf16<<<(I+255)/256,256,0,c->stream>>>((__nv_bfloat16*)c->dga1,c->dx,I);CUtensorMap a,b,d;
if(!dg_map(&a,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dga1,I,1,(unsigned long long)I*2,64,64,CU_TENSOR_MAP_SWIZZLE_128B)||!dg_map(&b,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,head->w,I,O,(unsigned long long)I*2,64,128,CU_TENSOR_MAP_SWIZZLE_128B)||!dg_map(&d,CU_TENSOR_MAP_DATA_TYPE_BFLOAT16,c->dgmm1,O,1,(unsigned long long)O*2,64,64,CU_TENSOR_MAP_SWIZZLE_128B))return 0;
- auto gemm=deep_gemm::sm120_bf16_gemm_impl<0,O,I,1,64,128,64,128,128,128,3,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,true,2>;if(!ok(cudaFuncSetAttribute(gemm,cudaFuncAttributeMaxDynamicSharedMemorySize,90368),"final head DeepGEMM shared memory"))return 0;
- gemm<<<170,384,90368,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(DgOutput*)c->dga1,(DgOutput*)head->w,nullptr,nullptr,1,O,I,a,b,d);dg_argmax_bf16<<<1,256,0,c->stream>>>(c->dgmm1,O,c->expert_ids,c->expert_weights);
+ auto gemm=c->sms>=170?deep_gemm::sm120_bf16_gemm_impl<0,O,I,1,64,128,64,128,128,128,3,128,256,170,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,true,2>:deep_gemm::sm120_bf16_gemm_impl<0,O,I,1,64,128,64,128,128,128,3,128,256,84,deep_gemm::GemmType::Normal,false,DgOutput,DgEpilogue,true,2>;if(!ok(cudaFuncSetAttribute(gemm,cudaFuncAttributeMaxDynamicSharedMemorySize,90368),"final head DeepGEMM shared memory"))return 0;
+ gemm<<<(c->sms>=170?170:84),384,90368,c->stream>>>((DgOutput*)c->dgmm1,nullptr,(DgOutput*)c->dga1,(DgOutput*)head->w,nullptr,nullptr,1,O,I,a,b,d);dg_argmax_bf16<<<1,256,0,c->stream>>>(c->dgmm1,O,c->expert_ids,c->expert_weights);
return ok(cudaGetLastError(),"final head launch")&&ok(cudaMemcpyAsync(id,c->expert_ids,sizeof(*id),cudaMemcpyDeviceToHost,c->stream),"final id download")&&ok(cudaMemcpyAsync(value,c->expert_weights,sizeof(*value),cudaMemcpyDeviceToHost,c->stream),"final logit download")&&ok(cudaStreamSynchronize(c->stream),"final head sync");
#else
(void)r;(void)fn;(void)scale;(void)base;(void)norm;(void)head;(void)M;(void)H;(void)eps;(void)pre_eps;(void)id;(void)value;return 0;
#endif
}
+#if COLI_DSV4_TC
static TcPlan *tc_plan(Dev*c,int O,int I){int pi=O>I;TcPlan*p=&c->plans[pi];if(p->ok)return p;if(p->op)plan_free(p);p->O=O;p->I=I;
cublasOperation_t ta=CUBLAS_OP_T,tb=CUBLAS_OP_N;cublasLtMatmulMatrixScale_t mode=CUBLASLT_MATMUL_MATRIX_SCALE_VEC32_UE8M0;
if(!bok(cublasLtMatmulDescCreate(&p->op,CUBLAS_COMPUTE_32F,CUDA_R_32F),"TC operation")||
@@ -1096,6 +1689,7 @@ static int tc_fp4_matvec(Dev*c,Dsv4CudaTensor*t,float*y){int O=t->O,I=t->I;size_
void *as=c->tcs,*bs=c->tcxs;if(!bok(cublasLtMatmulDescSetAttribute(p->op,CUBLASLT_MATMUL_DESC_A_SCALE_POINTER,&as,sizeof(as)),"TC A scale")||
!bok(cublasLtMatmulDescSetAttribute(p->op,CUBLASLT_MATMUL_DESC_B_SCALE_POINTER,&bs,sizeof(bs)),"TC B scale"))return 0;float alpha=1,beta=0;
return bok(cublasLtMatmul(c->lt,p->op,&alpha,c->tcw,p->a,c->tcx,p->b,&beta,y,p->c,y,p->d,&p->algo.algo,c->tc_workspace,32<<20,c->stream),"TC matvec");}
+#endif /* COLI_DSV4_TC */
extern "C" int dsv4_cuda_expert_group(Dsv4CudaTensor *const *gate,Dsv4CudaTensor *const *up,Dsv4CudaTensor *const *down,
const float *weights,int count,float limit,float *y,const float *x){
if(count<1||count>16||!gate||!up||!down)return 0;Dsv4CudaTensor *t=gate[0];Dev*c=t?ctx(t->device):nullptr;
@@ -1106,12 +1700,14 @@ extern "C" int dsv4_cuda_expert_group(Dsv4CudaTensor *const *gate,Dsv4CudaTensor
!buf((void**)&c->p1,&c->p1cap,ib)||!buf((void**)&c->p2,&c->p2cap,ib)||
!buf((void**)&c->p3,&c->p3cap,ib)||!buf((void**)&c->p4,&c->p4cap,hb))return 0;
if(!ok(cudaMemcpyAsync(c->dx,x,xb,cudaMemcpyHostToDevice,c->stream),"expert-group activation upload"))return 0;
+#if COLI_DSV4_TC
static int tc=-1;if(tc<0)tc=getenv("DSV4_CUDA_TC")?atoi(getenv("DSV4_CUDA_TC")):0;
if(tc){for(int e=0;e<count;e++){if(!tc_prepare_x(c,c->dx,H)||!tc_fp4_matvec(c,gate[e],c->p1+(long long)e*I)||!tc_fp4_matvec(c,up[e],c->p2+(long long)e*I))return 0;
expert_act<<<(I+255)/256,256,0,c->stream>>>(c->p3+(long long)e*I,c->p1+(long long)e*I,c->p2+(long long)e*I,weights[e],limit,I);
if(!tc_prepare_x(c,c->p3+(long long)e*I,I)||!tc_fp4_matvec(c,down[e],c->p4+(long long)e*H))return 0;}
expert_reduce<<<(H+255)/256,256,0,c->stream>>>(c->dy,c->p4,count,H);
return ok(cudaGetLastError(),"TC expert-group launch")&&ok(cudaMemcpyAsync(y,c->dy,(size_t)H*4,cudaMemcpyDeviceToHost,c->stream),"TC expert-group result download")&&ok(cudaStreamSynchronize(c->stream),"TC expert-group sync");}
+#endif /* COLI_DSV4_TC */
static int batched=-1;if(batched<0)batched=getenv("DSV4_CUDA_BATCHED")?atoi(getenv("DSV4_CUDA_BATCHED")):1;
if(batched){MvDesc hd[32];for(int e=0;e<count;e++){
hd[2*e]={(uint8_t*)gate[e]->w,gate[e]->scale,c->dx,c->p1+(long long)e*I};
@@ -1273,6 +1869,41 @@ extern "C" Dsv4CudaExpertSet *dsv4_cuda_expert_bank_create(int count,int H,int I
}
return set;
}
+/* ---- transparent host pinning for expert uploads ----
+ * The engine's expert-cache slot slabs are stable for the process lifetime
+ * (posix_memalign once per slot, freed only at store destroy), so the first
+ * upload from a slab page-locks its whole allocation region (VirtualQuery
+ * bounds) and every later upload from it is a pure DMA at PCIe speed instead
+ * of a CPU staging copy (~4.7 GB/s measured while the refill's parallel
+ * lookups compete for the cores). Failures fall back to pageable copies;
+ * DSV4_CUDA_PIN_HOST=0 disables; total pinned bytes are capped. */
+struct PinRegion{uintptr_t base;size_t size;};
+static std::vector<PinRegion> g_pin_regions,g_pin_failed;
+static size_t g_pin_bytes;
+static int g_pin_enabled=-1;
+static bool host_pinned(const void*p,size_t len){
+ if(g_pin_enabled<0){const char*e=getenv("DSV4_CUDA_PIN_HOST");g_pin_enabled=!(e&&*e=='0');}
+ if(!g_pin_enabled||!p||!len)return false;
+ uintptr_t a=(uintptr_t)p;
+ for(auto&r:g_pin_regions)if(a>=r.base&&a+len<=r.base+r.size)return true;
+ for(auto&r:g_pin_failed)if(a>=r.base&&a+len<=r.base+r.size)return false;
+#ifdef _WIN32
+ MEMORY_BASIC_INFORMATION mbi;
+ if(!VirtualQuery(p,&mbi,sizeof mbi)||mbi.State!=MEM_COMMIT)return false;
+ uintptr_t base=(uintptr_t)mbi.BaseAddress;size_t size=mbi.RegionSize;
+ if(a<base||a+len>base+size)return false;
+ static const size_t cap=(size_t)20<<30;
+ if(g_pin_bytes+size>cap){g_pin_failed.push_back({base,size});return false;}
+ cudaError_t err=cudaHostRegister((void*)base,size,cudaHostRegisterPortable);
+ if(err==cudaErrorHostMemoryAlreadyRegistered){cudaGetLastError();g_pin_regions.push_back({base,size});return true;}
+ if(err!=cudaSuccess){cudaGetLastError();g_pin_failed.push_back({base,size});return false;}
+ g_pin_bytes+=size;g_pin_regions.push_back({base,size});
+ return true;
+#else
+ (void)a;return false;
+#endif
+}
+
extern "C" int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet*set,int e,
const uint8_t*gw,const uint8_t*gs,const uint8_t*uw,const uint8_t*us,const uint8_t*dw,const uint8_t*ds,
Dsv4CudaTensor**gate,Dsv4CudaTensor**up,Dsv4CudaTensor**down){
@@ -1281,6 +1912,20 @@ extern "C" int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet*set,int e,
int H=set->H,I=set->I;size_t w1=(size_t)I*(H/2),s1=(size_t)I*(H/32),w2=(size_t)H*(I/2),s2=(size_t)H*(I/32);
uint8_t*fc1=set->bank_fc1+(size_t)e*2*w1,*fc1s=set->bank_fc1_scale+(size_t)e*2*s1;
uint8_t*fc2=set->bank_fc2+(size_t)e*w2,*fc2s=set->bank_fc2_scale+(size_t)e*s2;
+ Dev*c=ctx(set->device);
+ bool pinned=c&&host_pinned(gw,w1)&&host_pinned(uw,w1)&&host_pinned(gs,s1)&&
+ host_pinned(us,s1)&&host_pinned(dw,w2)&&host_pinned(ds,s2);
+ if(pinned){
+ /* Pure DMA on the device stream; the caller releases the host slab
+ * right after we return, so drain before returning. */
+ if(!ok(cudaMemcpyAsync(fc1,gw,w1,cudaMemcpyHostToDevice,c->stream),"expert gate upload")||
+ !ok(cudaMemcpyAsync(fc1+w1,uw,w1,cudaMemcpyHostToDevice,c->stream),"expert up upload")||
+ !ok(cudaMemcpyAsync(fc1s,gs,s1,cudaMemcpyHostToDevice,c->stream),"expert gate scale upload")||
+ !ok(cudaMemcpyAsync(fc1s+s1,us,s1,cudaMemcpyHostToDevice,c->stream),"expert up scale upload")||
+ !ok(cudaMemcpyAsync(fc2,dw,w2,cudaMemcpyHostToDevice,c->stream),"expert down upload")||
+ !ok(cudaMemcpyAsync(fc2s,ds,s2,cudaMemcpyHostToDevice,c->stream),"expert down scale upload")||
+ !ok(cudaStreamSynchronize(c->stream),"expert upload drain"))return 0;
+ }else
if(!ok(cudaMemcpy(fc1,gw,w1,cudaMemcpyHostToDevice),"expert gate upload")||
!ok(cudaMemcpy(fc1+w1,uw,w1,cudaMemcpyHostToDevice),"expert up upload")||
!ok(cudaMemcpy(fc1s,gs,s1,cudaMemcpyHostToDevice),"expert gate scale upload")||
@@ -1294,7 +1939,7 @@ extern "C" int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet*set,int e,
#endif
Dsv4CudaTensor*g=expert_view(fc1,fc1s,I,H,set->device),*u=expert_view(fc1+w1,fc1s+s1,I,H,set->device),*d=expert_view(fc2,fc2s,H,I,set->device);
if(!g||!u||!d){free(g);free(u);free(d);return 0;}
- ExpertPtr p[3]={{(uint8_t*)g->w,g->scale,nullptr},{(uint8_t*)u->w,u->scale,nullptr},{(uint8_t*)d->w,d->scale,nullptr}};
+ ExpertPtr p[3]={{(uint8_t*)g->w,g->scale},{(uint8_t*)u->w,u->scale},{(uint8_t*)d->w,d->scale}};
if(!ok(cudaMemcpy(set->table+e,p,sizeof(*p),cudaMemcpyHostToDevice),"expert gate table upload")||
!ok(cudaMemcpy(set->table+set->count+e,p+1,sizeof(*p),cudaMemcpyHostToDevice),"expert up table upload")||
!ok(cudaMemcpy(set->table+2*set->count+e,p+2,sizeof(*p),cudaMemcpyHostToDevice),"expert down table upload")){
@@ -1324,6 +1969,19 @@ extern "C" int dsv4_cuda_expert_bank_upload_tp2(Dsv4CudaExpertSet*set,int e,int
return 1;
}
extern "C" void dsv4_cuda_expert_set_free(Dsv4CudaExpertSet*set){if(!set)return;cudaSetDevice(set->device);if(set->table)cudaFree(set->table);if(set->hash)cudaFree(set->hash);if(set->bank_fc1)cudaFree(set->bank_fc1);if(set->bank_fc1_scale)cudaFree(set->bank_fc1_scale);if(set->bank_fc2)cudaFree(set->bank_fc2);if(set->bank_fc2_scale)cudaFree(set->bank_fc2_scale);if(set->bank_fc1_dg_scale)cudaFree(set->bank_fc1_dg_scale);if(set->bank_fc2_dg_scale)cudaFree(set->bank_fc2_dg_scale);free(set);}
+/* Rebind the shared-expert mirrors on a bank set. The bank is a per-layer
+ * streaming buffer during batched prefill, but the shared experts differ per
+ * layer and are already resident as the layer's fp8 mirrors — rebinding them
+ * is free, re-creating the ~2 GiB bank per layer is not. Same shape contract
+ * as dsv4_cuda_expert_bank_create. */
+extern "C" int dsv4_cuda_expert_bank_set_shared(Dsv4CudaExpertSet*set,Dsv4CudaTensor*sg,Dsv4CudaTensor*su,Dsv4CudaTensor*sd){
+ if(!set)return 0;
+ if(!sg&&!su&&!sd){set->sg=set->su=set->sd=nullptr;return 1;}
+ int H=set->H,J=sg?sg->O:0;
+ if(!sg||!su||!sd||sg->device!=set->device||su->device!=set->device||sd->device!=set->device||
+ sg->fmt!=8||su->fmt!=8||sd->fmt!=8||sg->I!=H||su->I!=H||su->O!=J||sd->I!=J||sd->O!=H)return 0;
+ set->sg=sg;set->su=su;set->sd=sd;return 1;
+}
extern "C" int dsv4_cuda_expert_set_upload_hash(Dsv4CudaExpertSet*set,const int64_t*map,int vocab,int topk){
if(!set||!map||vocab<1||topk!=6||!ok(cudaSetDevice(set->device),"select hash router device"))return 0;
size_t bytes=(size_t)vocab*topk*sizeof(*map);if(set->hash)cudaFree(set->hash);set->hash=nullptr;set->vocab=0;set->topk=0;
@@ -1392,10 +2050,10 @@ extern "C" int dsv4_cuda_route_moe(const Dsv4CudaActivation*input,Dsv4CudaTensor
#endif
fp8_sim<<<(H+127)/128,128,0,c->stream>>>(c->dx,1,H);
build_moe_desc<<<1,K,0,c->stream>>>(c->mvdesc,set->table,c->expert_ids,c->dx,c->p1,c->p2,c->p3,c->p4,K,set->count,H,I);
-run_fp4_grouped(c->mvdesc,2*K,I,H,c->stream);
+ run_fp4_grouped(c->mvdesc,2*K,I,H,c->stream);
expert_act_grouped<<<(K*I+255)/256,256,0,c->stream>>>(c->p3,c->p1,c->p2,c->expert_weights,limit,K,I);
fp8_sim<<<K*((I+127)/128),128,0,c->stream>>>(c->p3,K,I);
-run_fp4_grouped(c->mvdesc+2*K,K,H,I,c->stream);
+ run_fp4_grouped(c->mvdesc+2*K,K,H,I,c->stream);
expert_reduce<<<(H+255)/256,256,0,c->stream>>>(output->data,c->p4,K,H);
run_mv<8>((uint8_t*)set->sg->w,set->sg->scale,c->dx,c->p1,I,H,1,c->stream);
run_mv<8>((uint8_t*)set->su->w,set->su->scale,c->dx,c->p2,I,H,1,c->stream);
@@ -1405,21 +2063,129 @@ run_fp4_grouped(c->mvdesc+2*K,K,H,I,c->stream);
}
extern "C" int dsv4_cuda_route_moe_batch(const Dsv4CudaActivation*input,Dsv4CudaTensor*gate,Dsv4CudaTensor*bias,const int*tokens,int count,float scale,Dsv4CudaExpertSet*set,float limit,Dsv4CudaActivation*output){
#ifdef COLI_DSV4_DEEPGEMM
- Dev*c=input?ctx(input->device):nullptr;if(!c||!gate||!set||!output||count<1||count>64||set->device!=input->device||output->device!=input->device||gate->fmt!=32||gate->O!=256||gate->I!=4096||gate->device!=input->device||set->count!=256||set->H!=4096||set->I!=2048||input->elements<(long long)count*4096||output->elements<(long long)count*4096||(bias&&(bias->fmt!=32||bias->O*bias->I<256||bias->device!=input->device))||(set->hash&&!tokens)||!ok(cudaSetDevice(input->device),"select batched MoE device"))return 0;
+ Dev*c=input?ctx(input->device):nullptr;if(!c||!gate||!set||!output||count<1||count>128||set->device!=input->device||output->device!=input->device||gate->fmt!=32||gate->O!=256||gate->I!=4096||gate->device!=input->device||set->count!=256||set->H!=4096||set->I!=2048||input->elements<(long long)count*4096||output->elements<(long long)count*4096||(bias&&(bias->fmt!=32||bias->O*bias->I<256||bias->device!=input->device))||(set->hash&&!tokens)||!ok(cudaSetDevice(input->device),"select batched MoE device"))return 0;
int routes=count*6;size_t logits=(size_t)count*256*sizeof(float),ids=(size_t)routes*sizeof(int),weights=(size_t)routes*sizeof(float);
if(!buf((void**)&c->dy,&c->ycap,logits)||!buf((void**)&c->expert_ids,&c->expert_idcap,ids)||!buf((void**)&c->expert_weights,&c->expert_weightscap,weights)||
(set->hash&&!buf((void**)&c->aux3,&c->aux3cap,(size_t)count*sizeof(int))))return 0;
route_logits_batch<<<count*256,256,0,c->stream>>>((float*)gate->w,input->data,c->dy,count,256,4096);
if(set->hash){if(!ok(cudaMemcpyAsync(c->aux3,tokens,(size_t)count*sizeof(int),cudaMemcpyHostToDevice,c->stream),"batch token upload"))return 0;route_hash_batch<<<(routes+255)/256,256,0,c->stream>>>(c->expert_ids,set->hash,(int*)c->aux3,count);route_fixed_weights_batch<<<count,6,0,c->stream>>>(c->expert_weights,c->expert_ids,c->dy,count,scale);}
else route_top6_batch<<<count,32,0,c->stream>>>(c->expert_ids,c->expert_weights,bias?(float*)bias->w:nullptr,c->dy,count,scale);
+ /* DSV4_CUDA_MOE_PROF=1: wall-time the three segments (forced syncs, so
+ * only for diagnosis) and print the first few calls. */
+ static int prof=-1;static int prof_calls;if(prof<0){const char*e=getenv("DSV4_CUDA_MOE_PROF");prof=e&&atoi(e);}
+ double t_route=0,t_expert=0,t_shared=0;std::chrono::steady_clock::time_point ta,tb;
+ if(prof&&prof_calls<6){cudaStreamSynchronize(c->stream);ta=std::chrono::steady_clock::now();}
if(!dg_moe_batch_contiguous(c,set,input->data,c->expert_ids,c->expert_weights,count,limit,output->data))return 0;
+ if(prof&&prof_calls<6){cudaStreamSynchronize(c->stream);tb=std::chrono::steady_clock::now();t_expert=std::chrono::duration<double>(tb-ta).count();ta=tb;}
Dsv4CudaTensor*sg=set->sg,*su=set->su,*sd=set->sd;if(!sg||!su||!sd||sg->O!=2048||su->O!=2048||sd->I!=2048)return 0;
size_t mid=(size_t)count*2048*sizeof(float),hb=(size_t)count*4096*sizeof(float);if(!buf((void**)&c->p1,&c->p1cap,mid)||!buf((void**)&c->p2,&c->p2cap,mid)||!buf((void**)&c->p3,&c->p3cap,mid)||!buf((void**)&c->p4,&c->p4cap,hb))return 0;
- if(!dg_dense_batch_dispatch(c,sg,input->data,c->p1,count)||!dg_dense_batch_dispatch(c,su,input->data,c->p2,count))return 0;swiglu_rows<<<((long long)count*2048+255)/256,256,0,c->stream>>>(c->p3,c->p1,c->p2,limit,count,2048);if(!dg_dense_batch_dispatch(c,sd,c->p3,c->p4,count))return 0;add_rows<<<((long long)count*4096+255)/256,256,0,c->stream>>>(output->data,c->p4,count*4096);return ok(cudaGetLastError(),"batched MoE launch");
+ if(!dg_dense_batch_dispatch(c,sg,input->data,c->p1,count)||!dg_dense_batch_dispatch(c,su,input->data,c->p2,count))return 0;swiglu_rows<<<((long long)count*2048+255)/256,256,0,c->stream>>>(c->p3,c->p1,c->p2,limit,count,2048);if(!dg_dense_batch_dispatch(c,sd,c->p3,c->p4,count))return 0;add_rows<<<((long long)count*4096+255)/256,256,0,c->stream>>>(output->data,c->p4,count*4096);
+ if(prof&&prof_calls<6){cudaStreamSynchronize(c->stream);tb=std::chrono::steady_clock::now();t_shared=std::chrono::duration<double>(tb-ta).count();
+ fprintf(stderr,"[DSV4 CUDA] moe-batch prof count=%d experts=%.3fs shared=%.3fs\n",count,t_expert,t_shared);prof_calls++;}
+ (void)t_route;return ok(cudaGetLastError(),"batched MoE launch");
#else
(void)input;(void)gate;(void)bias;(void)tokens;(void)count;(void)scale;(void)set;(void)limit;(void)output;return 0;
#endif
}
+/* Batched top-6 routing only: logits from the f32 gate mirror, ids/weights
+ * downloaded to the host so the caller can stage exactly the routed experts
+ * before running the expert GEMMs. */
+extern "C" int dsv4_cuda_route_top6_batch(const Dsv4CudaActivation*input,Dsv4CudaTensor*gate,Dsv4CudaTensor*bias,int count,float scale,int*ids,float*weights){
+ Dev*c=input?ctx(input->device):nullptr;
+ if(!c||!gate||!ids||!weights||count<1||count>128||gate->fmt!=32||gate->O!=256||gate->I!=4096||gate->device!=input->device||
+ input->elements<(long long)count*4096||(bias&&(bias->fmt!=32||bias->O*bias->I<256||bias->device!=input->device))||
+ !ok(cudaSetDevice(input->device),"select batched route device"))return 0;
+ int routes=count*6;size_t logits=(size_t)count*256*sizeof(float),ib=(size_t)routes*sizeof(int),wb=(size_t)routes*sizeof(float);
+ if(!buf((void**)&c->dy,&c->ycap,logits)||!buf((void**)&c->expert_ids,&c->expert_idcap,ib)||!buf((void**)&c->expert_weights,&c->expert_weightscap,wb))return 0;
+ route_logits_batch<<<count*256,256,0,c->stream>>>((float*)gate->w,input->data,c->dy,count,256,4096);
+ route_top6_batch<<<count,32,0,c->stream>>>(c->expert_ids,c->expert_weights,bias?(float*)bias->w:nullptr,c->dy,count,scale);
+ return ok(cudaGetLastError(),"batched route launch")&&
+ ok(cudaMemcpyAsync(ids,c->expert_ids,ib,cudaMemcpyDeviceToHost,c->stream),"batched route id download")&&
+ ok(cudaMemcpyAsync(weights,c->expert_weights,wb,cudaMemcpyDeviceToHost,c->stream),"batched route weight download")&&
+ ok(cudaStreamSynchronize(c->stream),"batched route sync");
+}
+/* Batched MoE over host-provided routes (ids/weights per token, 6 each).
+ * The caller guarantees every id's bank slot holds that expert's weights;
+ * routed sum and the shared expert land in output like route_moe_batch. */
+extern "C" int dsv4_cuda_route_moe_ids_batch(const Dsv4CudaActivation*input,const int*ids,const float*weights,int count,Dsv4CudaExpertSet*set,float limit,Dsv4CudaActivation*output){
+#ifdef COLI_DSV4_DEEPGEMM
+ Dev*c=input?ctx(input->device):nullptr;
+ if(!c||!ids||!weights||!set||!output||count<1||count>128||set->device!=input->device||output->device!=input->device||
+ set->count!=256||set->H!=4096||set->I!=2048||!set->bank_fc1||
+ input->elements<(long long)count*4096||output->elements<(long long)count*4096||
+ !ok(cudaSetDevice(input->device),"select batched MoE device"))return 0;
+ int routes=count*6;
+ for(int r=0;r<routes;r++)if(ids[r]<0||ids[r]>=set->count)return 0;
+ size_t ib=(size_t)routes*sizeof(int),wb=(size_t)routes*sizeof(float);
+ if(!buf((void**)&c->expert_ids,&c->expert_idcap,ib)||!buf((void**)&c->expert_weights,&c->expert_weightscap,wb))return 0;
+ if(!ok(cudaMemcpyAsync(c->expert_ids,ids,ib,cudaMemcpyHostToDevice,c->stream),"batched MoE id upload")||
+ !ok(cudaMemcpyAsync(c->expert_weights,weights,wb,cudaMemcpyHostToDevice,c->stream),"batched MoE weight upload"))return 0;
+ static int prof=-1;static int prof_calls;if(prof<0){const char*e=getenv("DSV4_CUDA_MOE_PROF");prof=e&&atoi(e);}
+ double t_expert=0,t_shared=0;std::chrono::steady_clock::time_point ta,tb;
+ if(prof&&prof_calls<6){cudaStreamSynchronize(c->stream);ta=std::chrono::steady_clock::now();}
+ if(!dg_moe_batch_contiguous(c,set,input->data,c->expert_ids,c->expert_weights,count,limit,output->data,ids))return 0;
+ if(prof&&prof_calls<6){cudaStreamSynchronize(c->stream);tb=std::chrono::steady_clock::now();t_expert=std::chrono::duration<double>(tb-ta).count();ta=tb;}
+ Dsv4CudaTensor*sg=set->sg,*su=set->su,*sd=set->sd;if(!sg||!su||!sd||sg->O!=2048||su->O!=2048||sd->I!=2048)return 0;
+ size_t mid=(size_t)count*2048*sizeof(float),hb=(size_t)count*4096*sizeof(float);
+ if(!buf((void**)&c->p1,&c->p1cap,mid)||!buf((void**)&c->p2,&c->p2cap,mid)||!buf((void**)&c->p3,&c->p3cap,mid)||!buf((void**)&c->p4,&c->p4cap,hb))return 0;
+ if(!dg_dense_batch_dispatch(c,sg,input->data,c->p1,count)||!dg_dense_batch_dispatch(c,su,input->data,c->p2,count))return 0;
+ swiglu_rows<<<((long long)count*2048+255)/256,256,0,c->stream>>>(c->p3,c->p1,c->p2,limit,count,2048);
+ if(!dg_dense_batch_dispatch(c,sd,c->p3,c->p4,count))return 0;
+ add_rows<<<((long long)count*4096+255)/256,256,0,c->stream>>>(output->data,c->p4,count*4096);
+ if(prof&&prof_calls<6){cudaStreamSynchronize(c->stream);tb=std::chrono::steady_clock::now();t_shared=std::chrono::duration<double>(tb-ta).count();
+ fprintf(stderr,"[DSV4 CUDA] moe-ids prof count=%d experts=%.3fs shared=%.3fs\n",count,t_expert,t_shared);prof_calls++;}
+ return ok(cudaGetLastError(),"batched MoE ids launch");
+#else
+ /* GENERIC batched MoE (no DeepGEMM): the decode kernels applied to every
+ * token of the chunk in one grouped launch each — fp32 accumulate over
+ * fp4-decoded expert rows from the same VRAM bank, then the fp8 shared
+ * experts through the generic batch matmul. Slower than the tensor-core
+ * path (~10-20 ms per 128-token chunk-layer) but runs on any sm_80+ GPU,
+ * so prefill stays GPU-side instead of falling back to the CPU union. */
+ Dev*c=input?ctx(input->device):nullptr;
+ if(!c||!ids||!weights||!set||!output||count<1||count>128||set->device!=input->device||output->device!=input->device||
+ set->count!=256||set->H!=4096||set->I!=2048||!set->bank_fc1||
+ input->elements<(long long)count*4096||output->elements<(long long)count*4096||
+ !ok(cudaSetDevice(input->device),"select generic batched MoE device"))return 0;
+ const int H=4096,I=2048,K=6;int routes=count*K;
+ for(int r=0;r<routes;r++)if(ids[r]<0||ids[r]>=set->count)return 0;
+ Dsv4CudaTensor*sg=set->sg,*su=set->su,*sd=set->sd;
+ if(!sg||!su||!sd||sg->fmt!=8||su->fmt!=8||sd->fmt!=8||sg->O!=I||su->O!=I||sd->I!=I||sg->I!=H||sd->O!=H)return 0;
+ size_t w1=(size_t)I*(H/2),s1=(size_t)I*(H/32),w2=(size_t)H*(I/2),s2=(size_t)H*(I/32);
+ size_t rb=(size_t)routes*I*sizeof(float),pb=(size_t)routes*H*sizeof(float),wb=(size_t)routes*sizeof(float);
+ if(!buf((void**)&c->p1,&c->p1cap,rb)||!buf((void**)&c->p2,&c->p2cap,rb)||!buf((void**)&c->p3,&c->p3cap,rb)||
+ !buf((void**)&c->p4,&c->p4cap,pb)||!buf((void**)&c->expert_weights,&c->expert_weightscap,wb)||
+ !buf((void**)&c->mvdesc,&c->mvdesccap,(size_t)routes*2*sizeof(MvDesc)))return 0;
+ if(!ok(cudaMemcpyAsync(c->expert_weights,weights,wb,cudaMemcpyHostToDevice,c->stream),"generic MoE weight upload"))return 0;
+ std::vector<MvDesc> hd((size_t)routes*2);
+ for(int r=0;r<routes;r++){int e=ids[r],t=r/K;
+ const uint8_t*fc1=set->bank_fc1+(size_t)e*2*w1,*fc1s=set->bank_fc1_scale+(size_t)e*2*s1;
+ const float*x=input->data+(long long)t*H;
+ hd[2*r]={fc1,fc1s,x,c->p1+(long long)r*I};
+ hd[2*r+1]={fc1+w1,fc1s+s1,x,c->p2+(long long)r*I};}
+ if(!ok(cudaMemcpyAsync(c->mvdesc,hd.data(),(size_t)routes*2*sizeof(MvDesc),cudaMemcpyHostToDevice,c->stream),"generic MoE fc1 descriptors"))return 0;
+ mv_fp4_grouped<4><<<dim3((I+3)/4,routes*2),256,0,c->stream>>>(c->mvdesc,routes*2,I,H);
+ expert_act_grouped<<<((long long)routes*I+255)/256,256,0,c->stream>>>(c->p3,c->p1,c->p2,c->expert_weights,limit,routes,I);
+ fp8_sim<<<routes*((I+127)/128),128,0,c->stream>>>(c->p3,routes,I);
+ /* down: descriptors must not be overwritten while fc1 still reads them —
+ * the copy is stream-ordered after the launches above, so reuse is safe. */
+ for(int r=0;r<routes;r++){int e=ids[r];
+ hd[r]={set->bank_fc2+(size_t)e*w2,set->bank_fc2_scale+(size_t)e*s2,c->p3+(long long)r*I,c->p4+(long long)r*H};}
+ if(!ok(cudaMemcpyAsync(c->mvdesc,hd.data(),(size_t)routes*sizeof(MvDesc),cudaMemcpyHostToDevice,c->stream),"generic MoE down descriptors"))return 0;
+ mv_fp4_grouped<4><<<dim3((H+3)/4,routes),256,0,c->stream>>>(c->mvdesc,routes,H,I);
+ expert_reduce_rows<<<((long long)count*H+255)/256,256,0,c->stream>>>(output->data,c->p4,count,K,H);
+ /* shared experts (fp8 dense) over the whole chunk, then the same combine
+ * as decode: routed = bf16(routed + bf16(shared)). */
+ size_t mid=(size_t)count*I*sizeof(float),hb=(size_t)count*H*sizeof(float);
+ if(!buf((void**)&c->p1,&c->p1cap,mid)||!buf((void**)&c->p2,&c->p2cap,mid)||!buf((void**)&c->p3,&c->p3cap,mid)||!buf((void**)&c->p4,&c->p4cap,hb))return 0;
+ run_mm_batch(sg,input->data,c->p1,count,c->stream);run_mm_batch(su,input->data,c->p2,count,c->stream);
+ expert_act_rows1<<<((long long)count*I+255)/256,256,0,c->stream>>>(c->p3,c->p1,c->p2,limit,count*I);
+ fp8_sim<<<count*((I+127)/128),128,0,c->stream>>>(c->p3,count,I);
+ run_mm_batch(sd,c->p3,c->p4,count,c->stream);
+ moe_combine<<<((long long)count*H+255)/256,256,0,c->stream>>>(output->data,c->p4,count*H);
+ return ok(cudaGetLastError(),"generic batched MoE launch");
+#endif
+}
extern "C" int dsv4_cuda_route_moe_ep2(const Dsv4CudaActivation*input,Dsv4CudaTensor*gate,Dsv4CudaTensor*bias,
const Dsv4CudaActivation*peer_input,Dsv4CudaTensor*peer_gate,Dsv4CudaTensor*peer_bias,int token,float scale,
Dsv4CudaExpertSet*local,Dsv4CudaExpertSet*peer,float limit,Dsv4CudaActivation*output,Dsv4CudaActivation*peer_output){
@@ -1542,3 +2308,4 @@ extern "C" int dsv4_cuda_wo(Dsv4CudaTensor *wa,Dsv4CudaTensor *wb,int groups,flo
}
extern "C" void dsv4_cuda_tensor_free(Dsv4CudaTensor*t){if(!t)return;cudaSetDevice(t->device);if(t->own_w&&t->w)cudaFree(t->w);if(t->bf16_cache)cudaFree(t->bf16_cache);if(t->own_scale&&t->scale)cudaFree(t->scale);if(t->own_dg_scale&&t->dg_scale)cudaFree(t->dg_scale);free(t);}
extern "C" long long dsv4_cuda_tensor_bytes(const Dsv4CudaTensor*t){return t?t->bytes:0;}
+extern "C" int dsv4_cuda_tensor_device(const Dsv4CudaTensor*t){return t?t->device:-1;}
diff --git a/c/backend_cuda_dsv4.h b/c/backend_cuda_dsv4.h
index f41cc7b6..5566bf66 100644
--- a/c/backend_cuda_dsv4.h
+++ b/c/backend_cuda_dsv4.h
@@ -19,10 +19,55 @@ int dsv4_cuda_upload_fp8(Dsv4CudaTensor **t,const uint8_t *w,const uint8_t *scal
int dsv4_cuda_upload_fp8_bf16(Dsv4CudaTensor **t,const uint8_t *w,const uint8_t *scale,int O,int I,int device);
int dsv4_cuda_upload_fp4(Dsv4CudaTensor **t,const uint8_t *w,const uint8_t *scale,int O,int I,int device);
int dsv4_cuda_upload_bf16(Dsv4CudaTensor **t,const uint16_t *w,int O,int I,int device);
+/* Same-shape in-place refill of an fp4 mirror (no realloc); sync=1 drains. */
+int dsv4_cuda_tensor_refill_fp4(Dsv4CudaTensor *t,const uint8_t *w,const uint8_t *scale,int O,int I,int sync);
int dsv4_cuda_upload_f32(Dsv4CudaTensor **t,const float *w,int O,int I,int device);
int dsv4_cuda_matvec(Dsv4CudaTensor *t,float *y,const float *x);
int dsv4_cuda_matmul_batch(Dsv4CudaTensor *t,const Dsv4CudaActivation *input,
int tokens,Dsv4CudaActivation *output);
+int dsv4_cuda_matmul_bf16_batch(Dsv4CudaTensor *t,const float *x,int tokens,float *y);
+/* Host-pointer batched sparse window attention (prefill). vals holds
+ * [0,comp_base) window rows in position order then the compressed rows;
+ * meta[3t]=window row offset, meta[3t+1]=window rows, meta[3t+2]=compressed
+ * prefix visible to token t. out receives [tokens][heads*dim]. */
+int dsv4_cuda_sparse_attn_batch(int device,const float *q,const float *vals,
+ const float *sinks,const int *meta,int value_rows,
+ int comp_base,int heads,int dim,int tokens,
+ float scale,float *out);
+/* chunk holds the current chunk's kv rows (tokens x dim) starting at absolute
+ * position chunk_start; the engine appends them to the ring only AFTER the
+ * call (ring aliasing past window positions). */
+int dsv4_cuda_sparse_attn_batch_cached(int device,int layer,const float *q,
+ const float *chunk,int chunk_start,
+ const float *sinks,const int *meta,int abs_base,
+ int comp_limit,int heads,int dim,int tokens,
+ float scale,float *out);
+/* Indexed cached variant: meta[3t+2] is the SELECTED count and sel holds each
+ * token's compressed ordinals (tokens x selstride, score order). */
+int dsv4_cuda_sparse_attn_batch_cached_idx(int device,int layer,const float *q,
+ const float *chunk,int chunk_start,
+ const float *sinks,const int *meta,
+ const int *sel,int selstride,int abs_base,
+ int comp_limit,int heads,int dim,int tokens,
+ float scale,float *out);
+/* Batched indexer scoring: scores[t][c] = sum_h relu(q_th . k_c) * w_th for
+ * c < counts[t] (else 0), sequential fp32 order identical to the CPU loop. */
+int dsv4_cuda_indexer_score_batch(int device,const float *queries,const float *keys,
+ const float *head_w,const int *counts,int tokens,
+ int heads,int dim,int count,float *scores);
+/* Bitwise replica of the CPU reference fp8 matmul (row-major e4m3 weights,
+ * 128x128 block f32 scales, host-quantized activations). y[tokens][rows]. */
+int dsv4_cuda_fp8_ref_matmul(int device,const uint8_t *w,const float *bscale,
+ int rows,int cols,int packed_rows8,const float *x,
+ int tokens,float *y);
+/* Build/GPU compatibility (loader DLL selection). */
+int dsv4_cuda_backend_arch_ok(int device);
+const char *dsv4_cuda_backend_name(void);
+long long dsv4_cuda_mem_free_mb(int device);
+int dsv4_cuda_kv_ring_append(int device,int layer,const float *rows,int start_pos,
+ int count,int window,int dim);
+int dsv4_cuda_kv_comp_append(int device,int layer,const float *rows,int start_idx,
+ int count,int dim);
int dsv4_cuda_head_argmax(Dsv4CudaTensor *t,const float *x,int *id,float *value);
int dsv4_cuda_final_argmax(const Dsv4CudaActivation *residual,Dsv4CudaTensor *fn,Dsv4CudaTensor *scale,
Dsv4CudaTensor *base,Dsv4CudaTensor *norm,Dsv4CudaTensor *head,
@@ -43,6 +88,7 @@ int dsv4_cuda_wo(Dsv4CudaTensor *wo_a,Dsv4CudaTensor *wo_b,int groups,
float *out,const float *context);
void dsv4_cuda_tensor_free(Dsv4CudaTensor *t);
long long dsv4_cuda_tensor_bytes(const Dsv4CudaTensor *t);
+int dsv4_cuda_tensor_device(const Dsv4CudaTensor *t);
Dsv4CudaActivation *dsv4_cuda_activation_create(int device,long long elements);
void dsv4_cuda_activation_free(Dsv4CudaActivation *a);
int dsv4_cuda_activation_upload(Dsv4CudaActivation *a,const float *x,long long elements);
@@ -145,6 +191,8 @@ int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet *set,int expert,
const uint8_t *up_weight,const uint8_t *up_scale,
const uint8_t *down_weight,const uint8_t *down_scale,
Dsv4CudaTensor **gate,Dsv4CudaTensor **up,Dsv4CudaTensor **down);
+int dsv4_cuda_expert_bank_set_shared(Dsv4CudaExpertSet *set,Dsv4CudaTensor *sg,
+ Dsv4CudaTensor *su,Dsv4CudaTensor *sd);
int dsv4_cuda_expert_bank_upload_tp2(Dsv4CudaExpertSet *set,int expert,int rank,
const uint8_t *gate_weight,const uint8_t *gate_scale,
const uint8_t *up_weight,const uint8_t *up_scale,
@@ -157,6 +205,15 @@ int dsv4_cuda_route_moe(const Dsv4CudaActivation *input,Dsv4CudaTensor *gate,Dsv
int dsv4_cuda_route_moe_batch(const Dsv4CudaActivation *input,Dsv4CudaTensor *gate,
Dsv4CudaTensor *bias,const int *tokens,int count,float routed_scale,
Dsv4CudaExpertSet *experts,float limit,Dsv4CudaActivation *output);
+/* Batched top-6 routing only; ids[count*6] / weights[count*6] land on the host. */
+int dsv4_cuda_route_top6_batch(const Dsv4CudaActivation *input,Dsv4CudaTensor *gate,
+ Dsv4CudaTensor *bias,int count,float routed_scale,
+ int *ids,float *weights);
+/* Batched MoE over host-provided routes; every id's bank slot must already
+ * hold that expert (route-aware refill). */
+int dsv4_cuda_route_moe_ids_batch(const Dsv4CudaActivation *input,const int *ids,
+ const float *weights,int count,Dsv4CudaExpertSet *experts,
+ float limit,Dsv4CudaActivation *output);
int dsv4_cuda_route_moe_ep2(const Dsv4CudaActivation *input,Dsv4CudaTensor *gate,Dsv4CudaTensor *bias,
const Dsv4CudaActivation *peer_input,Dsv4CudaTensor *peer_gate,Dsv4CudaTensor *peer_bias,
int token,float routed_scale,Dsv4CudaExpertSet *local,Dsv4CudaExpertSet *peer,
diff --git a/c/backend_loader_dsv4.c b/c/backend_loader_dsv4.c
new file mode 100644
index 00000000..76d12f37
--- /dev/null
+++ b/c/backend_loader_dsv4.c
@@ -0,0 +1,1006 @@
+/* backend_loader_dsv4.c — Windows runtime loader for the DeepSeek V4 CUDA backend DLL.
+ *
+ * Why this exists: deepseek-v4.exe is built with MinGW-w64 (gcc), but the
+ * DeepSeek V4 CUDA kernels in backend_cuda_dsv4.cu must be compiled with
+ * MSVC + nvcc. The MSVC/MinGW ABI split means we cannot link a CUDA .o into a
+ * gcc binary, so the kernels are built into a standalone coli_cuda_dsv4.dll
+ * (nvcc + MSVC, symbols exported via dsv4.def) and resolved here at runtime
+ * through LoadLibrary/GetProcAddress. The host never links cudart directly.
+ *
+ * This mirrors backend_loader.c, which does the same for the GLM engine's
+ * coli_cuda.dll. The two loaders are separate files on purpose: the dsv4 ABI
+ * (dsv4_cuda_*) is entirely different from the GLM ABI (coli_cuda_*), and the
+ * dsv4 host is a different executable that must be able to load the V4 backend
+ * even when the GLM backend is absent.
+ *
+ * On non-Windows builds this file is not compiled (the Makefile.deepseek-v4
+ * links backend_cuda_dsv4.o directly). If the DLL is absent, every wrapper
+ * safely returns the "not initialized" sentinel and the engine falls back to
+ * CPU — the same contract the raw backend_cuda_dsv4.h functions have when no
+ * CUDA device is present.
+ *
+ * ABI note: Dsv4CudaTensor/Dsv4CudaActivation/Dsv4CudaKvCache/Dsv4CudaExpertSet/
+ * Dsv4CudaGraph are opaque to the host (it stores the pointer, never dereferences
+ * it), so the MSVC-allocated structs are safe to pass across the boundary. All
+ * scalar types (int, float, pointers, long long) agree between MSVC and
+ * MinGW-w64 on x86-64.
+ */
+#ifdef _WIN32
+
+#include <stdio.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <windows.h>
+
+#include "backend_cuda_dsv4.h"
+
+/* Two builds of the same source ship side by side: coli_cuda_dsv4_dg.dll
+ * (DeepGEMM, sm_120a tensor-core paths) and coli_cuda_dsv4.dll (portable
+ * generic kernels, sm_80+). The loader tries the DeepGEMM one first and asks
+ * it whether the device fits (dsv4_cuda_backend_arch_ok); on refusal it
+ * unloads and takes the generic one, so one install adapts to whatever GPU
+ * is present. COLI_DSV4_DLL=<file name> forces a specific DLL. */
+#define DSV4_BACKEND_DLL "coli_cuda_dsv4.dll"
+#define DSV4_BACKEND_DLL_DG "coli_cuda_dsv4_dg.dll"
+#define DSV4_VENDOR_TAG "[DSV4 CUDA]"
+
+/* Function-pointer typedefs matching each exported symbol in dsv4.def. */
+typedef int (*fn_init)(const int *devices, int count);
+typedef void (*fn_shutdown)(void);
+typedef Dsv4CudaActivation *(*fn_activation_create)(int device, long long elements);
+typedef void (*fn_activation_free)(Dsv4CudaActivation *a);
+typedef int (*fn_activation_upload)(Dsv4CudaActivation *a, const float *x, long long elements);
+typedef int (*fn_activation_download)(float *x, const Dsv4CudaActivation *a, long long elements);
+typedef int (*fn_activation_copy)(Dsv4CudaActivation *dst, const Dsv4CudaActivation *src, long long elements);
+typedef int (*fn_activation_copy_range)(Dsv4CudaActivation *dst, long long dst_offset,
+ const Dsv4CudaActivation *src, long long src_offset,
+ long long elements);
+typedef int (*fn_activation_sync)(const Dsv4CudaActivation *a);
+typedef int (*fn_activation_device)(const Dsv4CudaActivation *a);
+typedef int (*fn_decode_state_set)(int device, int token, int position);
+typedef void (*fn_profiler_start)(void);
+typedef void (*fn_profiler_stop)(void);
+typedef int (*fn_graph_begin)(int device);
+typedef Dsv4CudaGraph *(*fn_graph_end)(int device);
+typedef int (*fn_graph_end_pair)(int primary, int peer, Dsv4CudaGraph **primary_graph,
+ Dsv4CudaGraph **peer_graph);
+typedef int (*fn_graph_launch)(Dsv4CudaGraph *graph);
+typedef void (*fn_graph_free)(Dsv4CudaGraph *graph);
+typedef int (*fn_upload_fp8)(Dsv4CudaTensor **t, const uint8_t *w, const uint8_t *scale,
+ int O, int I, int device);
+typedef int (*fn_upload_fp8_bf16)(Dsv4CudaTensor **t, const uint8_t *w, const uint8_t *scale,
+ int O, int I, int device);
+typedef int (*fn_upload_fp4)(Dsv4CudaTensor **t, const uint8_t *w, const uint8_t *scale,
+ int O, int I, int device);
+typedef int (*fn_upload_bf16)(Dsv4CudaTensor **t, const uint16_t *w, int O, int I, int device);
+typedef int (*fn_upload_f32)(Dsv4CudaTensor **t, const float *w, int O, int I, int device);
+typedef int (*fn_mhc_pre)(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn,
+ Dsv4CudaTensor *scale, Dsv4CudaTensor *base, int M, int H,
+ float rms_eps, float pre_eps, float sink_eps, float post_mult,
+ int sink_iters, Dsv4CudaActivation *state, Dsv4CudaActivation *input);
+typedef int (*fn_mhc_post)(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ const Dsv4CudaActivation *state, int M, int H, Dsv4CudaActivation *out);
+typedef int (*fn_attention_first)(const Dsv4CudaActivation *input, Dsv4CudaTensor *attn_norm,
+ Dsv4CudaTensor *q_a, Dsv4CudaTensor *q_norm, Dsv4CudaTensor *q_b,
+ Dsv4CudaTensor *wkv, Dsv4CudaTensor *kv_norm, Dsv4CudaTensor *sink,
+ Dsv4CudaTensor *wo_a, Dsv4CudaTensor *wo_b, int heads, int head_dim,
+ int qk_rope, int groups, float eps, Dsv4CudaActivation *output);
+typedef Dsv4CudaKvCache *(*fn_kv_create)(int device, int window, int head_dim, int max_tokens,
+ int rope_pairs, const float *rope_cos, const float *rope_sin,
+ const float *compress_cos, const float *compress_sin);
+typedef int (*fn_mhc_pre_norm)(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn,
+ Dsv4CudaTensor *scale, Dsv4CudaTensor *base, Dsv4CudaTensor *norm,
+ int M, int H, float rms_eps, float pre_eps, float sink_eps,
+ float post_mult, int sink_iters, float norm_eps,
+ Dsv4CudaActivation *state, Dsv4CudaActivation *input);
+typedef int (*fn_mhc_pre_norm_batch)(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn,
+ Dsv4CudaTensor *scale, Dsv4CudaTensor *base, Dsv4CudaTensor *norm,
+ int tokens, int H, Dsv4CudaActivation *state, Dsv4CudaActivation *input);
+typedef int (*fn_mhc_pre_batch)(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn,
+ Dsv4CudaTensor *scale, Dsv4CudaTensor *base, int tokens, int H,
+ Dsv4CudaActivation *state, Dsv4CudaActivation *input);
+typedef int (*fn_mhc_post_pre)(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ Dsv4CudaActivation *state, int M, int H, Dsv4CudaActivation *out,
+ Dsv4CudaTensor *fn, Dsv4CudaTensor *scale, Dsv4CudaTensor *base,
+ float rms_eps, float pre_eps, float sink_eps, float post_mult,
+ int sink_iters, Dsv4CudaActivation *input);
+typedef int (*fn_mhc_post_pre_norm)(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ Dsv4CudaActivation *state, int M, int H, Dsv4CudaActivation *out,
+ Dsv4CudaTensor *fn, Dsv4CudaTensor *scale, Dsv4CudaTensor *base,
+ Dsv4CudaTensor *norm, float rms_eps, float pre_eps, float sink_eps,
+ float post_mult, int sink_iters, float norm_eps,
+ Dsv4CudaActivation *input);
+typedef int (*fn_mhc_post_pre_norm_batch)(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ Dsv4CudaActivation *state, int tokens, int H, Dsv4CudaActivation *out,
+ Dsv4CudaTensor *fn, Dsv4CudaTensor *scale, Dsv4CudaTensor *base,
+ Dsv4CudaTensor *norm, Dsv4CudaActivation *input);
+typedef int (*fn_mhc_post_batch)(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ const Dsv4CudaActivation *state, int tokens, int H,
+ Dsv4CudaActivation *out);
+typedef void (*fn_kv_free)(Dsv4CudaKvCache *cache);
+typedef int (*fn_attention_window)(const Dsv4CudaActivation *input, Dsv4CudaTensor *attn_norm,
+ Dsv4CudaTensor *q_a, Dsv4CudaTensor *q_norm, Dsv4CudaTensor *q_b,
+ Dsv4CudaTensor *wkv, Dsv4CudaTensor *kv_norm, Dsv4CudaTensor *sink,
+ Dsv4CudaTensor *wo_a, Dsv4CudaTensor *wo_b,
+ Dsv4CudaTensor *compress_wkv, Dsv4CudaTensor *compress_wgate,
+ Dsv4CudaTensor *compress_ape, Dsv4CudaTensor *compress_norm,
+ int compress_ratio, int heads, int head_dim, int qk_rope,
+ int groups, int pos, float eps, Dsv4CudaKvCache *cache,
+ Dsv4CudaActivation *output);
+typedef int (*fn_attention_sparse_batch)(const Dsv4CudaActivation *input, Dsv4CudaTensor *attn_norm,
+ Dsv4CudaTensor *qkv, Dsv4CudaTensor *q_norm, Dsv4CudaTensor *q_b,
+ Dsv4CudaTensor *kv_norm, Dsv4CudaTensor *sink, int heads,
+ int head_dim, int start_pos, int tokens, float eps,
+ Dsv4CudaKvCache *cache, Dsv4CudaActivation *context);
+typedef int (*fn_attention_output_batch)(const Dsv4CudaActivation *context, Dsv4CudaTensor *wo_a,
+ Dsv4CudaTensor *wo_b, int groups, int tokens,
+ Dsv4CudaActivation *output);
+typedef int (*fn_attention_window_tp2)(const Dsv4CudaActivation *input, Dsv4CudaActivation *peer_input,
+ const Dsv4CudaAttentionWeights *primary,
+ const Dsv4CudaAttentionWeights *peer, int compress_ratio,
+ int heads, int head_dim, int qk_rope, int groups, int pos,
+ float eps, Dsv4CudaKvCache *cache, Dsv4CudaKvCache *peer_cache,
+ Dsv4CudaActivation *output, Dsv4CudaActivation *peer_output);
+typedef int (*fn_route)(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate, Dsv4CudaTensor *bias,
+ const int *fixed_ids, float routed_scale, int ids[6], float weights[6]);
+typedef int (*fn_matvec_grouped)(Dsv4CudaTensor *t, float *y, const float *x, int groups);
+typedef int (*fn_matvec)(Dsv4CudaTensor *t, float *y, const float *x);
+typedef int (*fn_matmul_batch)(Dsv4CudaTensor *t, const Dsv4CudaActivation *input, int tokens,
+ Dsv4CudaActivation *output);
+typedef int (*fn_matmul_bf16_batch)(Dsv4CudaTensor *t, const float *x, int tokens, float *y);
+typedef int (*fn_sparse_attn_batch_cached)(int device, int layer, const float *q,
+ const float *chunk, int chunk_start,
+ const float *sinks, const int *meta,
+ int abs_base, int comp_limit, int heads,
+ int dim, int tokens, float scale, float *out);
+typedef int (*fn_sparse_attn_batch_cached_idx)(int device, int layer, const float *q,
+ const float *chunk, int chunk_start,
+ const float *sinks, const int *meta,
+ const int *sel, int selstride,
+ int abs_base, int comp_limit, int heads,
+ int dim, int tokens, float scale, float *out);
+typedef int (*fn_indexer_score_batch)(int device, const float *queries, const float *keys,
+ const float *head_w, const int *counts, int tokens,
+ int heads, int dim, int count, float *scores);
+typedef int (*fn_fp8_ref_matmul)(int device, const uint8_t *w, const float *bscale,
+ int rows, int cols, int packed_rows8, const float *x,
+ int tokens, float *y);
+typedef int (*fn_tensor_refill_fp4)(Dsv4CudaTensor *t, const uint8_t *w, const uint8_t *scale,
+ int O, int I, int sync);
+typedef int (*fn_backend_arch_ok)(int device);
+typedef const char *(*fn_backend_name)(void);
+typedef long long (*fn_mem_free_mb)(int device);
+typedef int (*fn_kv_ring_append)(int device, int layer, const float *rows,
+ int start_pos, int count, int window, int dim);
+typedef int (*fn_kv_comp_append)(int device, int layer, const float *rows,
+ int start_idx, int count, int dim);
+typedef int (*fn_sparse_attn_batch)(int device, const float *q, const float *vals,
+ const float *sinks, const int *meta, int value_rows,
+ int comp_base, int heads, int dim, int tokens,
+ float scale, float *out);
+typedef int (*fn_head_argmax)(Dsv4CudaTensor *t, const float *x, int *id, float *value);
+typedef int (*fn_final_argmax)(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn,
+ Dsv4CudaTensor *scale, Dsv4CudaTensor *base, Dsv4CudaTensor *norm,
+ Dsv4CudaTensor *head, int M, int H, float eps, float pre_eps,
+ int *id, float *value);
+typedef int (*fn_expert_group)(Dsv4CudaTensor *const *gate, Dsv4CudaTensor *const *up,
+ Dsv4CudaTensor *const *down, const float *weights, int count,
+ float limit, float *y, const float *x);
+typedef int (*fn_expert_fp8)(Dsv4CudaTensor *gate, Dsv4CudaTensor *up, Dsv4CudaTensor *down,
+ float limit, float *y, const float *x);
+typedef int (*fn_moe)(Dsv4CudaTensor *const *gate, Dsv4CudaTensor *const *up,
+ Dsv4CudaTensor *const *down, const float *weights, int count,
+ Dsv4CudaTensor *shared_gate, Dsv4CudaTensor *shared_up,
+ Dsv4CudaTensor *shared_down, float limit, float *y, const float *x);
+typedef int (*fn_rmsnorm)(Dsv4CudaActivation *x, Dsv4CudaTensor *weight, float eps, int elements);
+typedef int (*fn_moe_activation)(Dsv4CudaTensor *const *gate, Dsv4CudaTensor *const *up,
+ Dsv4CudaTensor *const *down, const float *weights, int count,
+ Dsv4CudaTensor *shared_gate, Dsv4CudaTensor *shared_up,
+ Dsv4CudaTensor *shared_down, float limit,
+ const Dsv4CudaActivation *input, Dsv4CudaActivation *output);
+typedef Dsv4CudaExpertSet *(*fn_expert_set_create)(Dsv4CudaTensor *const *gate, Dsv4CudaTensor *const *up,
+ Dsv4CudaTensor *const *down, int count,
+ Dsv4CudaTensor *shared_gate, Dsv4CudaTensor *shared_up,
+ Dsv4CudaTensor *shared_down);
+typedef Dsv4CudaExpertSet *(*fn_expert_bank_create)(int count, int hidden, int intermediate, int device,
+ Dsv4CudaTensor *shared_gate, Dsv4CudaTensor *shared_up,
+ Dsv4CudaTensor *shared_down);
+typedef int (*fn_expert_bank_upload)(Dsv4CudaExpertSet *set, int expert,
+ const uint8_t *gate_weight, const uint8_t *gate_scale,
+ const uint8_t *up_weight, const uint8_t *up_scale,
+ const uint8_t *down_weight, const uint8_t *down_scale,
+ Dsv4CudaTensor **gate, Dsv4CudaTensor **up, Dsv4CudaTensor **down);
+typedef int (*fn_expert_bank_set_shared)(Dsv4CudaExpertSet *set, Dsv4CudaTensor *sg,
+ Dsv4CudaTensor *su, Dsv4CudaTensor *sd);
+typedef int (*fn_expert_bank_upload_tp2)(Dsv4CudaExpertSet *set, int expert, int rank,
+ const uint8_t *gate_weight, const uint8_t *gate_scale,
+ const uint8_t *up_weight, const uint8_t *up_scale,
+ const uint8_t *down_weight, const uint8_t *down_scale);
+typedef void (*fn_expert_set_free)(Dsv4CudaExpertSet *set);
+typedef int (*fn_expert_set_upload_hash)(Dsv4CudaExpertSet *set, const int64_t *map,
+ int vocab, int topk);
+typedef int (*fn_route_moe)(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate, Dsv4CudaTensor *bias,
+ int token, float routed_scale, Dsv4CudaExpertSet *experts,
+ float limit, Dsv4CudaActivation *output);
+typedef int (*fn_route_moe_batch)(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate,
+ Dsv4CudaTensor *bias, const int *tokens, int count,
+ float routed_scale, Dsv4CudaExpertSet *experts, float limit,
+ Dsv4CudaActivation *output);
+typedef int (*fn_route_top6_batch)(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate,
+ Dsv4CudaTensor *bias, int count, float routed_scale,
+ int *ids, float *weights);
+typedef int (*fn_route_moe_ids_batch)(const Dsv4CudaActivation *input, const int *ids,
+ const float *weights, int count,
+ Dsv4CudaExpertSet *experts, float limit,
+ Dsv4CudaActivation *output);
+typedef int (*fn_route_moe_ep2)(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate,
+ Dsv4CudaTensor *bias, const Dsv4CudaActivation *peer_input,
+ Dsv4CudaTensor *peer_gate, Dsv4CudaTensor *peer_bias, int token,
+ float routed_scale, Dsv4CudaExpertSet *local, Dsv4CudaExpertSet *peer,
+ float limit, Dsv4CudaActivation *output, Dsv4CudaActivation *peer_output);
+typedef int (*fn_qkv)(Dsv4CudaTensor *q_a, Dsv4CudaTensor *q_norm, Dsv4CudaTensor *q_b,
+ Dsv4CudaTensor *kv, float eps, float *q_out, float *kv_out, const float *x);
+typedef int (*fn_wo)(Dsv4CudaTensor *wo_a, Dsv4CudaTensor *wo_b, int groups,
+ float *out, const float *context);
+typedef void (*fn_tensor_free)(Dsv4CudaTensor *t);
+typedef long long (*fn_tensor_bytes)(const Dsv4CudaTensor *t);
+typedef int (*fn_tensor_device)(const Dsv4CudaTensor *t);
+
+/* Resolved pointers, plus a flag so we attempt the load at most once. */
+static struct {
+ int loaded; /* 1 = load attempted (success or fail), 0 = not yet */
+ int available; /* 1 = DLL loaded and all symbols resolved */
+ HMODULE dll;
+ fn_init init;
+ fn_shutdown shutdown;
+ fn_activation_create activation_create;
+ fn_activation_free activation_free;
+ fn_activation_upload activation_upload;
+ fn_activation_download activation_download;
+ fn_activation_copy activation_copy;
+ fn_activation_copy_range activation_copy_range;
+ fn_activation_sync activation_sync;
+ fn_activation_device activation_device;
+ fn_decode_state_set decode_state_set;
+ fn_profiler_start profiler_start;
+ fn_profiler_stop profiler_stop;
+ fn_graph_begin graph_begin;
+ fn_graph_end graph_end;
+ fn_graph_end_pair graph_end_pair;
+ fn_graph_launch graph_launch;
+ fn_graph_free graph_free;
+ fn_upload_fp8 upload_fp8;
+ fn_upload_fp8_bf16 upload_fp8_bf16;
+ fn_upload_fp4 upload_fp4;
+ fn_upload_bf16 upload_bf16;
+ fn_upload_f32 upload_f32;
+ fn_mhc_pre mhc_pre;
+ fn_mhc_post mhc_post;
+ fn_attention_first attention_first;
+ fn_kv_create kv_create;
+ fn_mhc_pre_norm mhc_pre_norm;
+ fn_mhc_pre_norm_batch mhc_pre_norm_batch;
+ fn_mhc_pre_batch mhc_pre_batch;
+ fn_mhc_post_pre mhc_post_pre;
+ fn_mhc_post_pre_norm mhc_post_pre_norm;
+ fn_mhc_post_pre_norm_batch mhc_post_pre_norm_batch;
+ fn_mhc_post_batch mhc_post_batch;
+ fn_kv_free kv_free;
+ fn_attention_window attention_window;
+ fn_attention_sparse_batch attention_sparse_batch;
+ fn_attention_output_batch attention_output_batch;
+ fn_attention_window_tp2 attention_window_tp2;
+ fn_route route;
+ fn_matvec_grouped matvec_grouped;
+ fn_matvec matvec;
+ fn_matmul_batch matmul_batch;
+ fn_matmul_bf16_batch matmul_bf16_batch;
+ fn_sparse_attn_batch sparse_attn_batch;
+ fn_sparse_attn_batch_cached sparse_attn_batch_cached;
+ fn_sparse_attn_batch_cached_idx sparse_attn_batch_cached_idx;
+ fn_indexer_score_batch indexer_score_batch;
+ fn_fp8_ref_matmul fp8_ref_matmul;
+ fn_tensor_refill_fp4 tensor_refill_fp4;
+ fn_backend_arch_ok backend_arch_ok; /* optional */
+ fn_backend_name backend_name; /* optional */
+ char loaded_name[64];
+ fn_mem_free_mb mem_free_mb;
+ fn_kv_ring_append kv_ring_append;
+ fn_kv_comp_append kv_comp_append;
+ fn_head_argmax head_argmax;
+ fn_final_argmax final_argmax;
+ fn_expert_group expert_group;
+ fn_expert_fp8 expert_fp8;
+ fn_moe moe;
+ fn_rmsnorm rmsnorm;
+ fn_moe_activation moe_activation;
+ fn_expert_set_create expert_set_create;
+ fn_expert_bank_create expert_bank_create;
+ fn_expert_bank_upload expert_bank_upload;
+ fn_expert_bank_set_shared expert_bank_set_shared;
+ fn_expert_bank_upload_tp2 expert_bank_upload_tp2;
+ fn_expert_set_free expert_set_free;
+ fn_expert_set_upload_hash expert_set_upload_hash;
+ fn_route_moe route_moe;
+ fn_route_moe_batch route_moe_batch;
+ fn_route_top6_batch route_top6_batch;
+ fn_route_moe_ids_batch route_moe_ids_batch;
+ fn_route_moe_ep2 route_moe_ep2;
+ fn_qkv qkv;
+ fn_wo wo;
+ fn_tensor_free tensor_free;
+ fn_tensor_bytes tensor_bytes;
+ fn_tensor_device tensor_device;
+} g_dsv4;
+
+/* One-shot loader: mirror of coli_cuda_load(). The DLL is loaded from the
+ * engine's OWN directory by absolute path — never a bare name — so the
+ * dependency search anchors to the trusted install directory and an attacker
+ * cannot plant a same-named DLL in the CWD (classic DLL hijacking). */
+static HMODULE dsv4_load_one(const char *name){
+ char dllpath[MAX_PATH];
+ HMODULE dll = NULL;
+ DWORD mn = GetModuleFileNameA(NULL, dllpath, (DWORD)sizeof(dllpath));
+ if(mn > 0 && mn < sizeof(dllpath)){
+ char *slash = strrchr(dllpath, '\\');
+ if(slash && (size_t)(slash + 1 - dllpath) + strlen(name) + 1 <= sizeof(dllpath)){
+ strcpy(slash + 1, name);
+ dll = LoadLibraryExA(dllpath, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
+ }
+ }
+ if(!dll)
+ /* Fallback by name, restricted to the app dir and System32 — never CWD. */
+ dll = LoadLibraryExA(name, NULL,
+ LOAD_LIBRARY_SEARCH_APPLICATION_DIR | LOAD_LIBRARY_SEARCH_SYSTEM32);
+ return dll;
+}
+
+static int dsv4_cuda_resolve(const char *dllname);
+
+static int dsv4_cuda_load(void){
+ if(g_dsv4.loaded) return g_dsv4.available;
+ g_dsv4.loaded = 1;
+ const char *forced = getenv("COLI_DSV4_DLL");
+ const char *candidates[3];
+ int n = 0;
+ if(forced && *forced) candidates[n++] = forced;
+ else { candidates[n++] = DSV4_BACKEND_DLL_DG; candidates[n++] = DSV4_BACKEND_DLL; }
+ for(int i = 0; i < n; i++){
+ g_dsv4.dll = dsv4_load_one(candidates[i]);
+ if(!g_dsv4.dll) continue;
+ if(!dsv4_cuda_resolve(candidates[i])) continue; /* unloaded inside */
+ /* Optional compatibility probe: a build whose kernels cannot run on
+ * device 0 says so, and the next candidate is tried. */
+ if(g_dsv4.backend_arch_ok && !g_dsv4.backend_arch_ok(0)){
+ fprintf(stderr, DSV4_VENDOR_TAG " %s does not fit this GPU; trying the next backend\n",
+ candidates[i]);
+ FreeLibrary(g_dsv4.dll); g_dsv4.dll = NULL;
+ memset(&g_dsv4, 0, sizeof(g_dsv4)); g_dsv4.loaded = 1;
+ continue;
+ }
+ snprintf(g_dsv4.loaded_name, sizeof(g_dsv4.loaded_name), "%s", candidates[i]);
+ fprintf(stderr, DSV4_VENDOR_TAG " backend=%s (%s)\n", candidates[i],
+ g_dsv4.backend_name ? g_dsv4.backend_name() : "unknown");
+ g_dsv4.available = 1;
+ return 1;
+ }
+ fprintf(stderr, DSV4_VENDOR_TAG " no usable backend DLL (" DSV4_BACKEND_DLL_DG " / "
+ DSV4_BACKEND_DLL "); GPU tier disabled (CPU path remains active).\n");
+ return 0;
+}
+
+static int dsv4_cuda_resolve(const char *dllname){
+ /* Mandatory symbol: missing backend means the DLL is unusable, so unload. */
+ #define RESOLVE(name, type) \
+ _Pragma("GCC diagnostic push") \
+ _Pragma("GCC diagnostic ignored \"-Wcast-function-type\"") \
+ g_dsv4.name = (type)GetProcAddress(g_dsv4.dll, "dsv4_cuda_" #name); \
+ _Pragma("GCC diagnostic pop") \
+ if(!g_dsv4.name){ \
+ fprintf(stderr, DSV4_VENDOR_TAG " %s missing symbol dsv4_cuda_" #name "\n", dllname); \
+ FreeLibrary(g_dsv4.dll); g_dsv4.dll = NULL; \
+ return 0; }
+
+ RESOLVE(init, fn_init);
+ RESOLVE(shutdown, fn_shutdown);
+ RESOLVE(activation_create, fn_activation_create);
+ RESOLVE(activation_free, fn_activation_free);
+ RESOLVE(activation_upload, fn_activation_upload);
+ RESOLVE(activation_download, fn_activation_download);
+ RESOLVE(activation_copy, fn_activation_copy);
+ RESOLVE(activation_copy_range, fn_activation_copy_range);
+ RESOLVE(activation_sync, fn_activation_sync);
+ RESOLVE(activation_device, fn_activation_device);
+ RESOLVE(decode_state_set, fn_decode_state_set);
+ RESOLVE(profiler_start, fn_profiler_start);
+ RESOLVE(profiler_stop, fn_profiler_stop);
+ RESOLVE(graph_begin, fn_graph_begin);
+ RESOLVE(graph_end, fn_graph_end);
+ RESOLVE(graph_end_pair, fn_graph_end_pair);
+ RESOLVE(graph_launch, fn_graph_launch);
+ RESOLVE(graph_free, fn_graph_free);
+ RESOLVE(upload_fp8, fn_upload_fp8);
+ RESOLVE(upload_fp8_bf16, fn_upload_fp8_bf16);
+ RESOLVE(upload_fp4, fn_upload_fp4);
+ RESOLVE(upload_bf16, fn_upload_bf16);
+ RESOLVE(upload_f32, fn_upload_f32);
+ RESOLVE(mhc_pre, fn_mhc_pre);
+ RESOLVE(mhc_post, fn_mhc_post);
+ RESOLVE(attention_first, fn_attention_first);
+ RESOLVE(kv_create, fn_kv_create);
+ RESOLVE(mhc_pre_norm, fn_mhc_pre_norm);
+ RESOLVE(mhc_pre_norm_batch, fn_mhc_pre_norm_batch);
+ RESOLVE(mhc_pre_batch, fn_mhc_pre_batch);
+ RESOLVE(mhc_post_pre, fn_mhc_post_pre);
+ RESOLVE(mhc_post_pre_norm, fn_mhc_post_pre_norm);
+ RESOLVE(mhc_post_pre_norm_batch, fn_mhc_post_pre_norm_batch);
+ RESOLVE(mhc_post_batch, fn_mhc_post_batch);
+ RESOLVE(kv_free, fn_kv_free);
+ RESOLVE(attention_window, fn_attention_window);
+ RESOLVE(attention_sparse_batch, fn_attention_sparse_batch);
+ RESOLVE(attention_output_batch, fn_attention_output_batch);
+ RESOLVE(attention_window_tp2, fn_attention_window_tp2);
+ RESOLVE(route, fn_route);
+ RESOLVE(matvec_grouped, fn_matvec_grouped);
+ RESOLVE(matvec, fn_matvec);
+ RESOLVE(matmul_batch, fn_matmul_batch);
+ RESOLVE(matmul_bf16_batch, fn_matmul_bf16_batch);
+ RESOLVE(sparse_attn_batch, fn_sparse_attn_batch);
+ RESOLVE(sparse_attn_batch_cached, fn_sparse_attn_batch_cached);
+ RESOLVE(sparse_attn_batch_cached_idx, fn_sparse_attn_batch_cached_idx);
+ RESOLVE(indexer_score_batch, fn_indexer_score_batch);
+ RESOLVE(fp8_ref_matmul, fn_fp8_ref_matmul);
+ RESOLVE(tensor_refill_fp4, fn_tensor_refill_fp4);
+ RESOLVE(mem_free_mb, fn_mem_free_mb);
+ RESOLVE(kv_ring_append, fn_kv_ring_append);
+ RESOLVE(kv_comp_append, fn_kv_comp_append);
+ RESOLVE(head_argmax, fn_head_argmax);
+ RESOLVE(final_argmax, fn_final_argmax);
+ RESOLVE(expert_group, fn_expert_group);
+ RESOLVE(expert_fp8, fn_expert_fp8);
+ RESOLVE(moe, fn_moe);
+ RESOLVE(rmsnorm, fn_rmsnorm);
+ RESOLVE(moe_activation, fn_moe_activation);
+ RESOLVE(expert_set_create, fn_expert_set_create);
+ RESOLVE(expert_bank_create, fn_expert_bank_create);
+ RESOLVE(expert_bank_upload, fn_expert_bank_upload);
+ RESOLVE(expert_bank_set_shared, fn_expert_bank_set_shared);
+ RESOLVE(expert_bank_upload_tp2, fn_expert_bank_upload_tp2);
+ RESOLVE(expert_set_free, fn_expert_set_free);
+ RESOLVE(expert_set_upload_hash, fn_expert_set_upload_hash);
+ RESOLVE(route_moe, fn_route_moe);
+ RESOLVE(route_moe_batch, fn_route_moe_batch);
+ RESOLVE(route_top6_batch, fn_route_top6_batch);
+ RESOLVE(route_moe_ids_batch, fn_route_moe_ids_batch);
+ RESOLVE(route_moe_ep2, fn_route_moe_ep2);
+ RESOLVE(qkv, fn_qkv);
+ RESOLVE(wo, fn_wo);
+ RESOLVE(tensor_free, fn_tensor_free);
+ RESOLVE(tensor_bytes, fn_tensor_bytes);
+ RESOLVE(tensor_device, fn_tensor_device);
+ #undef RESOLVE
+ /* Optional probes (older DLLs lack them). */
+ _Pragma("GCC diagnostic push")
+ _Pragma("GCC diagnostic ignored \"-Wcast-function-type\"")
+ g_dsv4.backend_arch_ok = (fn_backend_arch_ok)GetProcAddress(g_dsv4.dll, "dsv4_cuda_backend_arch_ok");
+ g_dsv4.backend_name = (fn_backend_name)GetProcAddress(g_dsv4.dll, "dsv4_cuda_backend_name");
+ _Pragma("GCC diagnostic pop")
+ return 1;
+}
+
+/* ---- Public wrappers: match backend_cuda_dsv4.h signatures exactly.
+ * Each forwards to the resolved pointer; if the DLL never loaded, return the
+ * "not initialized" result the engine already handles (0 / NULL), so the
+ * engine falls back to CPU. ---- */
+
+int dsv4_cuda_init(const int *devices, int count){
+ if(!dsv4_cuda_load()) return 0;
+ return g_dsv4.init(devices, count);
+}
+
+void dsv4_cuda_shutdown(void){
+ if(g_dsv4.available && g_dsv4.shutdown) g_dsv4.shutdown();
+ if(g_dsv4.dll){ FreeLibrary(g_dsv4.dll); g_dsv4.dll = NULL; }
+ g_dsv4.available = 0;
+}
+
+Dsv4CudaActivation *dsv4_cuda_activation_create(int device, long long elements){
+ if(!g_dsv4.available) return NULL;
+ return g_dsv4.activation_create(device, elements);
+}
+
+void dsv4_cuda_activation_free(Dsv4CudaActivation *a){
+ if(g_dsv4.available && g_dsv4.activation_free) g_dsv4.activation_free(a);
+}
+
+int dsv4_cuda_activation_upload(Dsv4CudaActivation *a, const float *x, long long elements){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.activation_upload(a, x, elements);
+}
+
+int dsv4_cuda_activation_download(float *x, const Dsv4CudaActivation *a, long long elements){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.activation_download(x, a, elements);
+}
+
+int dsv4_cuda_activation_copy(Dsv4CudaActivation *dst, const Dsv4CudaActivation *src, long long elements){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.activation_copy(dst, src, elements);
+}
+
+int dsv4_cuda_activation_copy_range(Dsv4CudaActivation *dst, long long dst_offset,
+ const Dsv4CudaActivation *src, long long src_offset,
+ long long elements){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.activation_copy_range(dst, dst_offset, src, src_offset, elements);
+}
+
+int dsv4_cuda_activation_sync(const Dsv4CudaActivation *a){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.activation_sync(a);
+}
+
+int dsv4_cuda_activation_device(const Dsv4CudaActivation *a){
+ if(!g_dsv4.available) return -1;
+ return g_dsv4.activation_device(a);
+}
+
+int dsv4_cuda_decode_state_set(int device, int token, int position){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.decode_state_set(device, token, position);
+}
+
+void dsv4_cuda_profiler_start(void){
+ if(g_dsv4.available && g_dsv4.profiler_start) g_dsv4.profiler_start();
+}
+
+void dsv4_cuda_profiler_stop(void){
+ if(g_dsv4.available && g_dsv4.profiler_stop) g_dsv4.profiler_stop();
+}
+
+int dsv4_cuda_graph_begin(int device){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.graph_begin(device);
+}
+
+Dsv4CudaGraph *dsv4_cuda_graph_end(int device){
+ if(!g_dsv4.available) return NULL;
+ return g_dsv4.graph_end(device);
+}
+
+int dsv4_cuda_graph_end_pair(int primary, int peer, Dsv4CudaGraph **primary_graph, Dsv4CudaGraph **peer_graph){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.graph_end_pair(primary, peer, primary_graph, peer_graph);
+}
+
+int dsv4_cuda_graph_launch(Dsv4CudaGraph *graph){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.graph_launch(graph);
+}
+
+void dsv4_cuda_graph_free(Dsv4CudaGraph *graph){
+ if(g_dsv4.available && g_dsv4.graph_free) g_dsv4.graph_free(graph);
+}
+
+int dsv4_cuda_upload_fp8(Dsv4CudaTensor **t, const uint8_t *w, const uint8_t *scale, int O, int I, int device){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.upload_fp8(t, w, scale, O, I, device);
+}
+
+int dsv4_cuda_upload_fp8_bf16(Dsv4CudaTensor **t, const uint8_t *w, const uint8_t *scale, int O, int I, int device){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.upload_fp8_bf16(t, w, scale, O, I, device);
+}
+
+int dsv4_cuda_upload_fp4(Dsv4CudaTensor **t, const uint8_t *w, const uint8_t *scale, int O, int I, int device){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.upload_fp4(t, w, scale, O, I, device);
+}
+
+int dsv4_cuda_upload_bf16(Dsv4CudaTensor **t, const uint16_t *w, int O, int I, int device){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.upload_bf16(t, w, O, I, device);
+}
+
+int dsv4_cuda_upload_f32(Dsv4CudaTensor **t, const float *w, int O, int I, int device){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.upload_f32(t, w, O, I, device);
+}
+
+int dsv4_cuda_mhc_pre(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn, Dsv4CudaTensor *scale,
+ Dsv4CudaTensor *base, int M, int H, float rms_eps, float pre_eps, float sink_eps,
+ float post_mult, int sink_iters, Dsv4CudaActivation *state, Dsv4CudaActivation *input){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_pre(residual, fn, scale, base, M, H, rms_eps, pre_eps, sink_eps, post_mult,
+ sink_iters, state, input);
+}
+
+int dsv4_cuda_mhc_post(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ const Dsv4CudaActivation *state, int M, int H, Dsv4CudaActivation *out){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_post(x, residual, state, M, H, out);
+}
+
+int dsv4_cuda_attention_first(const Dsv4CudaActivation *input, Dsv4CudaTensor *attn_norm,
+ Dsv4CudaTensor *q_a, Dsv4CudaTensor *q_norm, Dsv4CudaTensor *q_b,
+ Dsv4CudaTensor *wkv, Dsv4CudaTensor *kv_norm, Dsv4CudaTensor *sink,
+ Dsv4CudaTensor *wo_a, Dsv4CudaTensor *wo_b, int heads, int head_dim,
+ int qk_rope, int groups, float eps, Dsv4CudaActivation *output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.attention_first(input, attn_norm, q_a, q_norm, q_b, wkv, kv_norm, sink, wo_a, wo_b,
+ heads, head_dim, qk_rope, groups, eps, output);
+}
+
+Dsv4CudaKvCache *dsv4_cuda_kv_create(int device, int window, int head_dim, int max_tokens,
+ int rope_pairs, const float *rope_cos, const float *rope_sin,
+ const float *compress_cos, const float *compress_sin){
+ if(!g_dsv4.available) return NULL;
+ return g_dsv4.kv_create(device, window, head_dim, max_tokens, rope_pairs,
+ rope_cos, rope_sin, compress_cos, compress_sin);
+}
+
+int dsv4_cuda_mhc_pre_norm(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn, Dsv4CudaTensor *scale,
+ Dsv4CudaTensor *base, Dsv4CudaTensor *norm, int M, int H, float rms_eps,
+ float pre_eps, float sink_eps, float post_mult, int sink_iters, float norm_eps,
+ Dsv4CudaActivation *state, Dsv4CudaActivation *input){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_pre_norm(residual, fn, scale, base, norm, M, H, rms_eps, pre_eps, sink_eps,
+ post_mult, sink_iters, norm_eps, state, input);
+}
+
+int dsv4_cuda_mhc_pre_norm_batch(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn,
+ Dsv4CudaTensor *scale, Dsv4CudaTensor *base, Dsv4CudaTensor *norm,
+ int tokens, int H, Dsv4CudaActivation *state, Dsv4CudaActivation *input){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_pre_norm_batch(residual, fn, scale, base, norm, tokens, H, state, input);
+}
+
+int dsv4_cuda_mhc_pre_batch(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn, Dsv4CudaTensor *scale,
+ Dsv4CudaTensor *base, int tokens, int H, Dsv4CudaActivation *state,
+ Dsv4CudaActivation *input){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_pre_batch(residual, fn, scale, base, tokens, H, state, input);
+}
+
+int dsv4_cuda_mhc_post_pre(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ Dsv4CudaActivation *state, int M, int H, Dsv4CudaActivation *out,
+ Dsv4CudaTensor *fn, Dsv4CudaTensor *scale, Dsv4CudaTensor *base,
+ float rms_eps, float pre_eps, float sink_eps, float post_mult, int sink_iters,
+ Dsv4CudaActivation *input){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_post_pre(x, residual, state, M, H, out, fn, scale, base,
+ rms_eps, pre_eps, sink_eps, post_mult, sink_iters, input);
+}
+
+int dsv4_cuda_mhc_post_pre_norm(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ Dsv4CudaActivation *state, int M, int H, Dsv4CudaActivation *out,
+ Dsv4CudaTensor *fn, Dsv4CudaTensor *scale, Dsv4CudaTensor *base,
+ Dsv4CudaTensor *norm, float rms_eps, float pre_eps, float sink_eps,
+ float post_mult, int sink_iters, float norm_eps, Dsv4CudaActivation *input){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_post_pre_norm(x, residual, state, M, H, out, fn, scale, base, norm,
+ rms_eps, pre_eps, sink_eps, post_mult, sink_iters, norm_eps, input);
+}
+
+int dsv4_cuda_mhc_post_pre_norm_batch(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ Dsv4CudaActivation *state, int tokens, int H, Dsv4CudaActivation *out,
+ Dsv4CudaTensor *fn, Dsv4CudaTensor *scale, Dsv4CudaTensor *base,
+ Dsv4CudaTensor *norm, Dsv4CudaActivation *input){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_post_pre_norm_batch(x, residual, state, tokens, H, out, fn, scale, base, norm, input);
+}
+
+int dsv4_cuda_mhc_post_batch(const Dsv4CudaActivation *x, const Dsv4CudaActivation *residual,
+ const Dsv4CudaActivation *state, int tokens, int H, Dsv4CudaActivation *out){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.mhc_post_batch(x, residual, state, tokens, H, out);
+}
+
+void dsv4_cuda_kv_free(Dsv4CudaKvCache *cache){
+ if(g_dsv4.available && g_dsv4.kv_free) g_dsv4.kv_free(cache);
+}
+
+int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4CudaTensor *attn_norm,
+ Dsv4CudaTensor *q_a, Dsv4CudaTensor *q_norm, Dsv4CudaTensor *q_b,
+ Dsv4CudaTensor *wkv, Dsv4CudaTensor *kv_norm, Dsv4CudaTensor *sink,
+ Dsv4CudaTensor *wo_a, Dsv4CudaTensor *wo_b,
+ Dsv4CudaTensor *compress_wkv, Dsv4CudaTensor *compress_wgate,
+ Dsv4CudaTensor *compress_ape, Dsv4CudaTensor *compress_norm,
+ int compress_ratio, int heads, int head_dim, int qk_rope, int groups,
+ int pos, float eps, Dsv4CudaKvCache *cache, Dsv4CudaActivation *output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.attention_window(input, attn_norm, q_a, q_norm, q_b, wkv, kv_norm, sink, wo_a, wo_b,
+ compress_wkv, compress_wgate, compress_ape, compress_norm,
+ compress_ratio, heads, head_dim, qk_rope, groups, pos, eps, cache, output);
+}
+
+int dsv4_cuda_attention_sparse_batch(const Dsv4CudaActivation *input, Dsv4CudaTensor *attn_norm,
+ Dsv4CudaTensor *qkv, Dsv4CudaTensor *q_norm, Dsv4CudaTensor *q_b,
+ Dsv4CudaTensor *kv_norm, Dsv4CudaTensor *sink, int heads,
+ int head_dim, int start_pos, int tokens, float eps,
+ Dsv4CudaKvCache *cache, Dsv4CudaActivation *context){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.attention_sparse_batch(input, attn_norm, qkv, q_norm, q_b, kv_norm, sink, heads,
+ head_dim, start_pos, tokens, eps, cache, context);
+}
+
+int dsv4_cuda_attention_output_batch(const Dsv4CudaActivation *context, Dsv4CudaTensor *wo_a,
+ Dsv4CudaTensor *wo_b, int groups, int tokens, Dsv4CudaActivation *output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.attention_output_batch(context, wo_a, wo_b, groups, tokens, output);
+}
+
+int dsv4_cuda_attention_window_tp2(const Dsv4CudaActivation *input, Dsv4CudaActivation *peer_input,
+ const Dsv4CudaAttentionWeights *primary, const Dsv4CudaAttentionWeights *peer,
+ int compress_ratio, int heads, int head_dim, int qk_rope, int groups,
+ int pos, float eps, Dsv4CudaKvCache *cache, Dsv4CudaKvCache *peer_cache,
+ Dsv4CudaActivation *output, Dsv4CudaActivation *peer_output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.attention_window_tp2(input, peer_input, primary, peer, compress_ratio, heads, head_dim,
+ qk_rope, groups, pos, eps, cache, peer_cache, output, peer_output);
+}
+
+int dsv4_cuda_route(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate, Dsv4CudaTensor *bias,
+ const int *fixed_ids, float routed_scale, int ids[6], float weights[6]){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.route(input, gate, bias, fixed_ids, routed_scale, ids, weights);
+}
+
+int dsv4_cuda_matvec_grouped(Dsv4CudaTensor *t, float *y, const float *x, int groups){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.matvec_grouped(t, y, x, groups);
+}
+
+int dsv4_cuda_matvec(Dsv4CudaTensor *t, float *y, const float *x){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.matvec(t, y, x);
+}
+
+int dsv4_cuda_matmul_batch(Dsv4CudaTensor *t, const Dsv4CudaActivation *input, int tokens,
+ Dsv4CudaActivation *output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.matmul_batch(t, input, tokens, output);
+}
+
+int dsv4_cuda_matmul_bf16_batch(Dsv4CudaTensor *t, const float *x, int tokens, float *y){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.matmul_bf16_batch(t, x, tokens, y);
+}
+
+int dsv4_cuda_sparse_attn_batch(int device, const float *q, const float *vals, const float *sinks,
+ const int *meta, int value_rows, int comp_base, int heads, int dim,
+ int tokens, float scale, float *out){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.sparse_attn_batch(device, q, vals, sinks, meta, value_rows, comp_base,
+ heads, dim, tokens, scale, out);
+}
+
+int dsv4_cuda_sparse_attn_batch_cached(int device, int layer, const float *q, const float *chunk,
+ int chunk_start, const float *sinks,
+ const int *meta, int abs_base, int comp_limit, int heads,
+ int dim, int tokens, float scale, float *out){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.sparse_attn_batch_cached(device, layer, q, chunk, chunk_start, sinks, meta,
+ abs_base, comp_limit, heads, dim, tokens, scale, out);
+}
+
+int dsv4_cuda_sparse_attn_batch_cached_idx(int device, int layer, const float *q, const float *chunk,
+ int chunk_start, const float *sinks,
+ const int *meta, const int *sel, int selstride,
+ int abs_base, int comp_limit, int heads,
+ int dim, int tokens, float scale, float *out){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.sparse_attn_batch_cached_idx(device, layer, q, chunk, chunk_start, sinks, meta,
+ sel, selstride, abs_base, comp_limit, heads, dim,
+ tokens, scale, out);
+}
+
+int dsv4_cuda_indexer_score_batch(int device, const float *queries, const float *keys,
+ const float *head_w, const int *counts, int tokens,
+ int heads, int dim, int count, float *scores){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.indexer_score_batch(device, queries, keys, head_w, counts, tokens,
+ heads, dim, count, scores);
+}
+
+int dsv4_cuda_fp8_ref_matmul(int device, const uint8_t *w, const float *bscale,
+ int rows, int cols, int packed_rows8, const float *x,
+ int tokens, float *y){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.fp8_ref_matmul(device, w, bscale, rows, cols, packed_rows8, x, tokens, y);
+}
+
+int dsv4_cuda_tensor_refill_fp4(Dsv4CudaTensor *t, const uint8_t *w, const uint8_t *scale, int O, int I, int sync){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.tensor_refill_fp4(t, w, scale, O, I, sync);
+}
+
+int dsv4_cuda_backend_arch_ok(int device){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.backend_arch_ok ? g_dsv4.backend_arch_ok(device) : 1;
+}
+
+const char *dsv4_cuda_backend_name(void){
+ if(!g_dsv4.available) return "none";
+ return g_dsv4.backend_name ? g_dsv4.backend_name() : "unknown";
+}
+
+long long dsv4_cuda_mem_free_mb(int device){
+ if(!g_dsv4.available) return -1;
+ return g_dsv4.mem_free_mb(device);
+}
+
+int dsv4_cuda_kv_ring_append(int device, int layer, const float *rows, int start_pos,
+ int count, int window, int dim){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.kv_ring_append(device, layer, rows, start_pos, count, window, dim);
+}
+
+int dsv4_cuda_kv_comp_append(int device, int layer, const float *rows, int start_idx,
+ int count, int dim){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.kv_comp_append(device, layer, rows, start_idx, count, dim);
+}
+
+int dsv4_cuda_head_argmax(Dsv4CudaTensor *t, const float *x, int *id, float *value){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.head_argmax(t, x, id, value);
+}
+
+int dsv4_cuda_final_argmax(const Dsv4CudaActivation *residual, Dsv4CudaTensor *fn, Dsv4CudaTensor *scale,
+ Dsv4CudaTensor *base, Dsv4CudaTensor *norm, Dsv4CudaTensor *head, int M, int H,
+ float eps, float pre_eps, int *id, float *value){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.final_argmax(residual, fn, scale, base, norm, head, M, H, eps, pre_eps, id, value);
+}
+
+int dsv4_cuda_expert_group(Dsv4CudaTensor *const *gate, Dsv4CudaTensor *const *up,
+ Dsv4CudaTensor *const *down, const float *weights, int count, float limit,
+ float *y, const float *x){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.expert_group(gate, up, down, weights, count, limit, y, x);
+}
+
+int dsv4_cuda_expert_fp8(Dsv4CudaTensor *gate, Dsv4CudaTensor *up, Dsv4CudaTensor *down, float limit,
+ float *y, const float *x){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.expert_fp8(gate, up, down, limit, y, x);
+}
+
+int dsv4_cuda_moe(Dsv4CudaTensor *const *gate, Dsv4CudaTensor *const *up,
+ Dsv4CudaTensor *const *down, const float *weights, int count,
+ Dsv4CudaTensor *shared_gate, Dsv4CudaTensor *shared_up, Dsv4CudaTensor *shared_down,
+ float limit, float *y, const float *x){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.moe(gate, up, down, weights, count, shared_gate, shared_up, shared_down, limit, y, x);
+}
+
+int dsv4_cuda_rmsnorm(Dsv4CudaActivation *x, Dsv4CudaTensor *weight, float eps, int elements){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.rmsnorm(x, weight, eps, elements);
+}
+
+int dsv4_cuda_moe_activation(Dsv4CudaTensor *const *gate, Dsv4CudaTensor *const *up,
+ Dsv4CudaTensor *const *down, const float *weights, int count,
+ Dsv4CudaTensor *shared_gate, Dsv4CudaTensor *shared_up,
+ Dsv4CudaTensor *shared_down, float limit, const Dsv4CudaActivation *input,
+ Dsv4CudaActivation *output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.moe_activation(gate, up, down, weights, count, shared_gate, shared_up, shared_down,
+ limit, input, output);
+}
+
+Dsv4CudaExpertSet *dsv4_cuda_expert_set_create(Dsv4CudaTensor *const *gate, Dsv4CudaTensor *const *up,
+ Dsv4CudaTensor *const *down, int count,
+ Dsv4CudaTensor *shared_gate, Dsv4CudaTensor *shared_up,
+ Dsv4CudaTensor *shared_down){
+ if(!g_dsv4.available) return NULL;
+ return g_dsv4.expert_set_create(gate, up, down, count, shared_gate, shared_up, shared_down);
+}
+
+Dsv4CudaExpertSet *dsv4_cuda_expert_bank_create(int count, int hidden, int intermediate, int device,
+ Dsv4CudaTensor *shared_gate, Dsv4CudaTensor *shared_up,
+ Dsv4CudaTensor *shared_down){
+ if(!g_dsv4.available) return NULL;
+ return g_dsv4.expert_bank_create(count, hidden, intermediate, device, shared_gate, shared_up, shared_down);
+}
+
+int dsv4_cuda_expert_bank_upload(Dsv4CudaExpertSet *set, int expert,
+ const uint8_t *gate_weight, const uint8_t *gate_scale,
+ const uint8_t *up_weight, const uint8_t *up_scale,
+ const uint8_t *down_weight, const uint8_t *down_scale,
+ Dsv4CudaTensor **gate, Dsv4CudaTensor **up, Dsv4CudaTensor **down){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.expert_bank_upload(set, expert, gate_weight, gate_scale, up_weight, up_scale,
+ down_weight, down_scale, gate, up, down);
+}
+
+int dsv4_cuda_expert_bank_set_shared(Dsv4CudaExpertSet *set, Dsv4CudaTensor *sg,
+ Dsv4CudaTensor *su, Dsv4CudaTensor *sd){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.expert_bank_set_shared(set, sg, su, sd);
+}
+
+int dsv4_cuda_expert_bank_upload_tp2(Dsv4CudaExpertSet *set, int expert, int rank,
+ const uint8_t *gate_weight, const uint8_t *gate_scale,
+ const uint8_t *up_weight, const uint8_t *up_scale,
+ const uint8_t *down_weight, const uint8_t *down_scale){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.expert_bank_upload_tp2(set, expert, rank, gate_weight, gate_scale, up_weight, up_scale,
+ down_weight, down_scale);
+}
+
+void dsv4_cuda_expert_set_free(Dsv4CudaExpertSet *set){
+ if(g_dsv4.available && g_dsv4.expert_set_free) g_dsv4.expert_set_free(set);
+}
+
+int dsv4_cuda_expert_set_upload_hash(Dsv4CudaExpertSet *set, const int64_t *map, int vocab, int topk){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.expert_set_upload_hash(set, map, vocab, topk);
+}
+
+int dsv4_cuda_route_moe(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate, Dsv4CudaTensor *bias,
+ int token, float routed_scale, Dsv4CudaExpertSet *experts, float limit,
+ Dsv4CudaActivation *output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.route_moe(input, gate, bias, token, routed_scale, experts, limit, output);
+}
+
+int dsv4_cuda_route_moe_batch(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate, Dsv4CudaTensor *bias,
+ const int *tokens, int count, float routed_scale, Dsv4CudaExpertSet *experts,
+ float limit, Dsv4CudaActivation *output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.route_moe_batch(input, gate, bias, tokens, count, routed_scale, experts, limit, output);
+}
+
+int dsv4_cuda_route_top6_batch(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate, Dsv4CudaTensor *bias,
+ int count, float routed_scale, int *ids, float *weights){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.route_top6_batch(input, gate, bias, count, routed_scale, ids, weights);
+}
+
+int dsv4_cuda_route_moe_ids_batch(const Dsv4CudaActivation *input, const int *ids, const float *weights,
+ int count, Dsv4CudaExpertSet *experts, float limit,
+ Dsv4CudaActivation *output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.route_moe_ids_batch(input, ids, weights, count, experts, limit, output);
+}
+
+int dsv4_cuda_route_moe_ep2(const Dsv4CudaActivation *input, Dsv4CudaTensor *gate, Dsv4CudaTensor *bias,
+ const Dsv4CudaActivation *peer_input, Dsv4CudaTensor *peer_gate,
+ Dsv4CudaTensor *peer_bias, int token, float routed_scale,
+ Dsv4CudaExpertSet *local, Dsv4CudaExpertSet *peer, float limit,
+ Dsv4CudaActivation *output, Dsv4CudaActivation *peer_output){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.route_moe_ep2(input, gate, bias, peer_input, peer_gate, peer_bias, token, routed_scale,
+ local, peer, limit, output, peer_output);
+}
+
+int dsv4_cuda_qkv(Dsv4CudaTensor *q_a, Dsv4CudaTensor *q_norm, Dsv4CudaTensor *q_b, Dsv4CudaTensor *kv,
+ float eps, float *q_out, float *kv_out, const float *x){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.qkv(q_a, q_norm, q_b, kv, eps, q_out, kv_out, x);
+}
+
+int dsv4_cuda_wo(Dsv4CudaTensor *wo_a, Dsv4CudaTensor *wo_b, int groups, float *out, const float *context){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.wo(wo_a, wo_b, groups, out, context);
+}
+
+void dsv4_cuda_tensor_free(Dsv4CudaTensor *t){
+ if(g_dsv4.available && g_dsv4.tensor_free) g_dsv4.tensor_free(t);
+}
+
+long long dsv4_cuda_tensor_bytes(const Dsv4CudaTensor *t){
+ if(!g_dsv4.available) return 0;
+ return g_dsv4.tensor_bytes(t);
+}
+
+int dsv4_cuda_tensor_device(const Dsv4CudaTensor *t){
+ if(!g_dsv4.available) return -1;
+ return g_dsv4.tensor_device(t);
+}
+
+#endif /* _WIN32 */
diff --git a/c/dsv4.def b/c/dsv4.def
new file mode 100644
index 00000000..9407b10e
--- /dev/null
+++ b/c/dsv4.def
@@ -0,0 +1,80 @@
+EXPORTS
+ dsv4_cuda_init
+ dsv4_cuda_shutdown
+ dsv4_cuda_activation_create
+ dsv4_cuda_activation_free
+ dsv4_cuda_activation_upload
+ dsv4_cuda_activation_download
+ dsv4_cuda_activation_copy
+ dsv4_cuda_activation_copy_range
+ dsv4_cuda_activation_sync
+ dsv4_cuda_activation_device
+ dsv4_cuda_decode_state_set
+ dsv4_cuda_profiler_start
+ dsv4_cuda_profiler_stop
+ dsv4_cuda_graph_begin
+ dsv4_cuda_graph_end
+ dsv4_cuda_graph_end_pair
+ dsv4_cuda_graph_launch
+ dsv4_cuda_graph_free
+ dsv4_cuda_upload_fp8
+ dsv4_cuda_upload_fp8_bf16
+ dsv4_cuda_upload_fp4
+ dsv4_cuda_upload_bf16
+ dsv4_cuda_upload_f32
+ dsv4_cuda_mhc_pre
+ dsv4_cuda_mhc_post
+ dsv4_cuda_attention_first
+ dsv4_cuda_kv_create
+ dsv4_cuda_mhc_pre_norm
+ dsv4_cuda_mhc_pre_norm_batch
+ dsv4_cuda_mhc_pre_batch
+ dsv4_cuda_mhc_post_pre
+ dsv4_cuda_mhc_post_pre_norm
+ dsv4_cuda_mhc_post_pre_norm_batch
+ dsv4_cuda_mhc_post_batch
+ dsv4_cuda_kv_free
+ dsv4_cuda_attention_window
+ dsv4_cuda_attention_sparse_batch
+ dsv4_cuda_attention_output_batch
+ dsv4_cuda_attention_window_tp2
+ dsv4_cuda_route
+ dsv4_cuda_matvec_grouped
+ dsv4_cuda_matvec
+ dsv4_cuda_matmul_batch
+ dsv4_cuda_matmul_bf16_batch
+ dsv4_cuda_sparse_attn_batch
+ dsv4_cuda_sparse_attn_batch_cached
+ dsv4_cuda_sparse_attn_batch_cached_idx
+ dsv4_cuda_indexer_score_batch
+ dsv4_cuda_fp8_ref_matmul
+ dsv4_cuda_tensor_refill_fp4
+ dsv4_cuda_backend_arch_ok
+ dsv4_cuda_backend_name
+ dsv4_cuda_kv_ring_append
+ dsv4_cuda_mem_free_mb
+ dsv4_cuda_kv_comp_append
+ dsv4_cuda_head_argmax
+ dsv4_cuda_final_argmax
+ dsv4_cuda_expert_group
+ dsv4_cuda_expert_fp8
+ dsv4_cuda_moe
+ dsv4_cuda_rmsnorm
+ dsv4_cuda_moe_activation
+ dsv4_cuda_expert_set_create
+ dsv4_cuda_expert_bank_create
+ dsv4_cuda_expert_bank_upload
+ dsv4_cuda_expert_bank_set_shared
+ dsv4_cuda_expert_bank_upload_tp2
+ dsv4_cuda_expert_set_free
+ dsv4_cuda_expert_set_upload_hash
+ dsv4_cuda_route_moe
+ dsv4_cuda_route_moe_batch
+ dsv4_cuda_route_top6_batch
+ dsv4_cuda_route_moe_ids_batch
+ dsv4_cuda_route_moe_ep2
+ dsv4_cuda_qkv
+ dsv4_cuda_wo
+ dsv4_cuda_tensor_free
+ dsv4_cuda_tensor_bytes
+ dsv4_cuda_tensor_device
diff --git a/c/patches-deepgemm-sm120-msvc.patch b/c/patches-deepgemm-sm120-msvc.patch
new file mode 100644
index 00000000..e2181949
--- /dev/null
+++ b/c/patches-deepgemm-sm120-msvc.patch
@@ -0,0 +1,69 @@
+diff --git a/deep_gemm/include/deep_gemm/common/math.cuh b/deep_gemm/include/deep_gemm/common/math.cuh
+index 0f0d250..4d44ea7 100644
+--- a/deep_gemm/include/deep_gemm/common/math.cuh
++++ b/deep_gemm/include/deep_gemm/common/math.cuh
+@@ -63,9 +63,15 @@ CUTLASS_DEVICE float2 fma2(const float2& a, const float2& b, const float2& c) {
+ }
+
+ CUTLASS_HOST_DEVICE float fast_rcp(const float& x) {
++#ifdef __CUDA_ARCH__
+ float ret;
+ asm volatile("rcp.approx.ftz.f32 %0, %1;" : "=f"(ret) : "f"(x));
+ return ret;
++#else
++ /* MSVC x64 host pass has no inline asm; the host fallback only needs to
++ * be arithmetically reasonable, it is never on the kernel path. */
++ return 1.0f / x;
++#endif
+ }
+
+ /// Casting
+diff --git a/deep_gemm/include/deep_gemm/impls/sm120_fp8_fp4_gemm_1d1d.cuh b/deep_gemm/include/deep_gemm/impls/sm120_fp8_fp4_gemm_1d1d.cuh
+index 4f2d312..1b594ae 100644
+--- a/deep_gemm/include/deep_gemm/impls/sm120_fp8_fp4_gemm_1d1d.cuh
++++ b/deep_gemm/include/deep_gemm/impls/sm120_fp8_fp4_gemm_1d1d.cuh
+@@ -26,6 +26,11 @@
+
+ namespace deep_gemm {
+
++// See the tma_param_pad parameter note below: 12 x uint32_t = 48 bytes of
++// kernel-parameter padding so the by-value CUtensorMap descriptors start at a
++// 64-byte-aligned offset under MSVC's parameter layout too.
++struct TmaParamPad48 { uint32_t _pad[12]; };
++
+ template <uint32_t SHAPE_M, uint32_t SHAPE_N, uint32_t SHAPE_K,
+ uint32_t kGranKA, uint32_t kGranKB,
+ uint32_t kNumGroups,
+@@ -53,6 +58,14 @@ sm120_fp8_fp4_gemm_1d1d_impl(cd_dtype_t* gmem_d, const cd_dtype_t* gmem_c,
+ float* gmem_workspace,
+ uint32_t shape_m, uint32_t shape_n, uint32_t shape_k,
+ uint32_t stride_cd_m, uint32_t stride_cd_n, uint32_t stride_cd_batch,
++ /* Windows port: MSVC/nvcc lays out by-value kernel
++ * parameters without honouring CUtensorMap's
++ * alignas(64), so the __grid_constant__ descriptors
++ * below landed at +0x50 and every TMA access faulted
++ * (misaligned address). 48 bytes of explicit padding
++ * pushes them to +0x80; on Linux the compiler already
++ * padded to +0x80, so the layout is unchanged there. */
++ TmaParamPad48 tma_param_pad,
+ const __grid_constant__ cute::TmaDescriptor tensor_map_a_base,
+ const __grid_constant__ cute::TmaDescriptor tensor_map_b_base,
+ const __grid_constant__ cute::TmaDescriptor tensor_map_sfa,
+diff --git a/deep_gemm/include/deep_gemm/ptx/ld_st.cuh b/deep_gemm/include/deep_gemm/ptx/ld_st.cuh
+index 806a4c2..aa520bb 100644
+--- a/deep_gemm/include/deep_gemm/ptx/ld_st.cuh
++++ b/deep_gemm/include/deep_gemm/ptx/ld_st.cuh
+@@ -149,9 +149,13 @@ CUTLASS_DEVICE void st_shared(const void* ptr, uint32_t x, uint32_t y, uint32_t
+ asm volatile("st.shared.v4.u32 [%0], {%1, %2, %3, %4};" :: "l"(__cvta_generic_to_shared(ptr)), "r"(x), "r"(y), "r"(z), "r"(w));
+ }
+
++#ifndef _MSC_VER
++/* MSVC has no __int128; nothing in the sm120 impls instantiates this
++ * overload, so hide it from the MSVC host pass (Windows DLL build). */
+ CUTLASS_DEVICE void st_shared(const __int128_t* ptr, __int128_t val) {
+ asm volatile("st.shared.b128 [%0], %1;" :: "l"(__cvta_generic_to_shared(ptr)), "q"(val));
+ }
++#endif
+
+ CUTLASS_DEVICE uint32_t mapa_shared(const uint32_t& ptr, const uint32_t& dst_cta_idx) {
+ uint32_t mapped;
diff --git a/c/tests/test_backend_cuda_dsv4.cu b/c/tests/test_backend_cuda_dsv4.cu
index 1bbe9012..446805e9 100644
--- a/c/tests/test_backend_cuda_dsv4.cu
+++ b/c/tests/test_backend_cuda_dsv4.cu
@@ -54,10 +54,10 @@ int main(int argc,char**){int devices[6]={0,1,2,3,4,5},dev=0,ndev=argc>1?6:1;if(
if(!dsv4_cuda_qkv(tqa,tqn,tqb,tkv,1e-6f,q.data(),k.data(),nx.data()))return 10;for(int h=0;h<heads;h++){double ss=0;for(int d=0;d<dim;d++)ss+=(double)q[h*dim+d]*q[h*dim+d];float inv=1.f/sqrtf((float)(ss/dim)+1e-6f);for(int d=0;d<dim;d++)q[h*dim+d]=dsv4_q_bf16(q[h*dim+d]*inv);}double kss=0;for(float v:k)kss+=(double)v*v;float kinv=1.f/sqrtf((float)(kss/dim)+1e-6f);for(int d=0;d<dim;d++)k[d]=dsv4_q_bf16(k[d]*kinv);dsv4_fp8_simulate(k.data(),dim-rope,64);
for(int h=0;h<heads;h++){float score=0;for(int d=0;d<dim;d++)score+=q[h*dim+d]*k[d];score/=sqrtf((float)dim);float mx=fmaxf(score,sinks[h]),a=expf(score-mx),den=a+expf(sinks[h]-mx);for(int d=0;d<dim;d++)context[h*dim+d]=dsv4_q_bf16(a*k[d]/den);}if(!dsv4_cuda_wo(twa,twb,groups,attref.data(),context.data()))return 10;
Dsv4CudaActivation *ai=dsv4_cuda_activation_create(dev,H),*ay=dsv4_cuda_activation_create(dev,H);if(!ai||!ay||!dsv4_cuda_activation_upload(ai,x.data(),H)||!dsv4_cuda_attention_first(ai,tan,tqa,tqn,tqb,tkv,tkvn,tsink,twa,twb,heads,dim,rope,groups,1e-6f,ay)||!dsv4_cuda_activation_download(attgot.data(),ay,H)||!dsv4_cuda_activation_sync(ay))return 10;for(int i=0;i<H;i++)worst=fmaxf(worst,fabsf(attref[i]-attgot[i]));
- float rc[2*rp],rs[2*rp];for(int p=0;p<rp;p++){rc[p]=1;rs[p]=0;float a=.1f*(p+1);rc[rp+p]=cosf(a);rs[rp+p]=sinf(a);}Dsv4CudaKvCache *kc=dsv4_cuda_kv_create(dev,4,dim,2,rp,rc,rs,rc,rs);if(!kc||!dsv4_cuda_attention_window(ai,tan,tqa,tqn,tqb,tkv,tkvn,tsink,twa,twb,nullptr,nullptr,nullptr,nullptr,0,heads,dim,rope,groups,0,1e-6f,kc,ay)||!dsv4_cuda_activation_download(attgot.data(),ay,H)||!dsv4_cuda_activation_sync(ay))return 11;for(int i=0;i<H;i++)worst=fmaxf(worst,fabsf(attref[i]-attgot[i]));
+ float rc[2*rp],rs[2*rp];for(int p=0;p<rp;p++){rc[p]=1;rs[p]=0;float a=.1f*(p+1);rc[rp+p]=cosf(a);rs[rp+p]=sinf(a);}Dsv4CudaKvCache *kc=dsv4_cuda_kv_create(dev,4,dim,2,rp,rc,rs,rc,rs);if(!kc||!dsv4_cuda_decode_state_set(dev,0,0)||!dsv4_cuda_attention_window(ai,tan,tqa,tqn,tqb,tkv,tkvn,tsink,twa,twb,nullptr,nullptr,nullptr,nullptr,0,heads,dim,rope,groups,0,1e-6f,kc,ay)||!dsv4_cuda_activation_download(attgot.data(),ay,H)||!dsv4_cuda_activation_sync(ay))return 11;for(int i=0;i<H;i++)worst=fmaxf(worst,fabsf(attref[i]-attgot[i]));
std::vector<float>x2(H),nx2(H),q2(H),k2(dim),ctx2(H),ref2(H);for(int i=0;i<H;i++)x2[i]=x[i]+(i%5-2)*.0078125f;double ss2=0;for(float v:x2)ss2+=(double)v*v;float iv2=1.f/sqrtf((float)(ss2/H)+1e-6f);for(int i=0;i<H;i++)nx2[i]=dsv4_q_bf16(x2[i]*iv2);dsv4_fp8_simulate(nx2.data(),H,128);if(!dsv4_cuda_qkv(tqa,tqn,tqb,tkv,1e-6f,q2.data(),k2.data(),nx2.data()))return 11;
for(int h=0;h<heads;h++){double ss=0;for(int d=0;d<dim;d++)ss+=(double)q2[h*dim+d]*q2[h*dim+d];float inv=1.f/sqrtf((float)(ss/dim)+1e-6f);for(int d=0;d<dim;d++)q2[h*dim+d]=dsv4_q_bf16(q2[h*dim+d]*inv);for(int p=0;p<rp;p++){int o=h*dim+dim-rope+2*p;float u=q2[o],v=q2[o+1];q2[o]=dsv4_q_bf16(u*rc[rp+p]-v*rs[rp+p]);q2[o+1]=dsv4_q_bf16(u*rs[rp+p]+v*rc[rp+p]);}}double ks2=0;for(float v:k2)ks2+=(double)v*v;float ik2=1.f/sqrtf((float)(ks2/dim)+1e-6f);for(int d=0;d<dim;d++)k2[d]=dsv4_q_bf16(k2[d]*ik2);for(int p=0;p<rp;p++){int o=dim-rope+2*p;float u=k2[o],v=k2[o+1];k2[o]=dsv4_q_bf16(u*rc[rp+p]-v*rs[rp+p]);k2[o+1]=dsv4_q_bf16(u*rs[rp+p]+v*rc[rp+p]);}dsv4_fp8_simulate(k2.data(),dim-rope,64);
- for(int h=0;h<heads;h++){float z0=0,z1=0;for(int d=0;d<dim;d++){z0+=q2[h*dim+d]*k[d];z1+=q2[h*dim+d]*k2[d];}z0/=sqrtf((float)dim);z1/=sqrtf((float)dim);float mx=fmaxf(sinks[h],fmaxf(z0,z1)),a0=expf(z0-mx),a1=expf(z1-mx),den=expf(sinks[h]-mx)+a0+a1;for(int d=0;d<dim;d++)ctx2[h*dim+d]=dsv4_q_bf16((a0*k[d]+a1*k2[d])/den);for(int p=0;p<rp;p++){int o=h*dim+dim-rope+2*p;float u=ctx2[o],v=ctx2[o+1];ctx2[o]=dsv4_q_bf16(u*rc[rp+p]+v*rs[rp+p]);ctx2[o+1]=dsv4_q_bf16(-u*rs[rp+p]+v*rc[rp+p]);}}if(!dsv4_cuda_wo(twa,twb,groups,ref2.data(),ctx2.data())||!dsv4_cuda_activation_upload(ai,x2.data(),H)||!dsv4_cuda_attention_window(ai,tan,tqa,tqn,tqb,tkv,tkvn,tsink,twa,twb,nullptr,nullptr,nullptr,nullptr,0,heads,dim,rope,groups,1,1e-6f,kc,ay)||!dsv4_cuda_activation_download(attgot.data(),ay,H)||!dsv4_cuda_activation_sync(ay))return 11;for(int i=0;i<H;i++)worst=fmaxf(worst,fabsf(ref2[i]-attgot[i]));dsv4_cuda_kv_free(kc);
+ for(int h=0;h<heads;h++){float z0=0,z1=0;for(int d=0;d<dim;d++){z0+=q2[h*dim+d]*k[d];z1+=q2[h*dim+d]*k2[d];}z0/=sqrtf((float)dim);z1/=sqrtf((float)dim);float mx=fmaxf(sinks[h],fmaxf(z0,z1)),a0=expf(z0-mx),a1=expf(z1-mx),den=expf(sinks[h]-mx)+a0+a1;for(int d=0;d<dim;d++)ctx2[h*dim+d]=dsv4_q_bf16((a0*k[d]+a1*k2[d])/den);for(int p=0;p<rp;p++){int o=h*dim+dim-rope+2*p;float u=ctx2[o],v=ctx2[o+1];ctx2[o]=dsv4_q_bf16(u*rc[rp+p]+v*rs[rp+p]);ctx2[o+1]=dsv4_q_bf16(-u*rs[rp+p]+v*rc[rp+p]);}}if(!dsv4_cuda_wo(twa,twb,groups,ref2.data(),ctx2.data())||!dsv4_cuda_activation_upload(ai,x2.data(),H)||!dsv4_cuda_decode_state_set(dev,1,1)||!dsv4_cuda_attention_window(ai,tan,tqa,tqn,tqb,tkv,tkvn,tsink,twa,twb,nullptr,nullptr,nullptr,nullptr,0,heads,dim,rope,groups,1,1e-6f,kc,ay)||!dsv4_cuda_activation_download(attgot.data(),ay,H)||!dsv4_cuda_activation_sync(ay))return 11;for(int i=0;i<H;i++)worst=fmaxf(worst,fabsf(ref2[i]-attgot[i]));dsv4_cuda_kv_free(kc);
Dsv4CudaTensor *at[]={tan,tqa,tqn,tqb,tkv,tkvn,tsink,twa,twb};for(auto*t:at)dsv4_cuda_tensor_free(t);dsv4_cuda_activation_free(ai);dsv4_cuda_activation_free(ay);
std::vector<float>rg(256*H),rb(256);for(int i=0;i<256*H;i++)rg[i]=(i%31-15)*.0005f;for(int i=0;i<256;i++)rb[i]=(i%13-6)*.002f;Dsv4CudaTensor *trg=nullptr,*trb=nullptr;Dsv4CudaActivation *rax=dsv4_cuda_activation_create(dev,H);int rid[6],expect_id[6],fixed[6]={7,19,31,63,127,255};float rw[6],expect_w[6];
std::vector<float>rlog(256);if(!rax||!dsv4_cuda_upload_f32(&trg,rg.data(),256,H,dev)||!dsv4_cuda_upload_f32(&trb,rb.data(),256,1,dev)||!dsv4_cuda_matvec(trg,rlog.data(),x.data())||!dsv4_cuda_activation_upload(rax,x.data(),H)||!dsv4_cuda_route(rax,trg,trb,nullptr,1.5f,rid,rw))return 12;
diff --git a/c/tests/test_dsv4_decode_cuda.c b/c/tests/test_dsv4_decode_cuda.c
new file mode 100644
index 00000000..8e9914f9
--- /dev/null
+++ b/c/tests/test_dsv4_decode_cuda.c
@@ -0,0 +1,234 @@
+/* test_dsv4_decode_cuda.c — decode-path smoke test for the DeepSeek V4 CUDA
+ * kernels, intended to be linked against the RUNTIME LOADER
+ * (backend_loader_dsv4.c) to validate the Windows MSVC->MinGW ABI boundary,
+ * or against the raw nvcc object on Linux. Only BASE-BUILD kernels are used
+ * (the DeepGEMM/FlashInfer batched paths are build-option gated and return 0).
+ */
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "../backend_cuda_dsv4.h"
+
+static float fp8_decode(uint8_t b){
+ if(!(b & 0x7f)) return 0.0f;
+ int s=b&0x80,e=(b>>3)&0xf,m=b&7;
+ float v=ldexpf(1.0f+m/8.0f,e-7);
+ return s?-v:v;
+}
+
+static float e8m0_decode(uint8_t b){ return ldexpf(1.0f,b-127); }
+
+static float fp4_decode(uint8_t q){ static const float a[8]={0,.5f,1,1.5f,2,3,4,6}; return q&8?-a[q&7]:a[q]; }
+
+static int ref_matvec(float *y,const uint8_t*w,const uint8_t*s,int O,int I,const float*x){
+ for(int o=0;o<O;o++){double acc=0;for(int i=0;i<I;i++)acc+=fp8_decode(w[(size_t)o*I+i])*e8m0_decode(s[(o/128)*((I+127)/128)+(i/128)])*x[i];y[o]=(float)acc;}
+ return 1;
+}
+
+int main(void){
+ enum { T=4,H=4096,R=1024,K=512,HEADS=64,Q=HEADS*K,MAX=32,ROPE=32,GROUPS=8 };
+ int dev=0;
+ if(!dsv4_cuda_init(&dev,1)){fprintf(stderr,"init failed (loader/DLL problem?)\n");return 1;}
+
+ /* --- part 1: fp8 matvec against an exact CPU reference ----------------- */
+ {
+ enum { O=512,I=4096 };
+ uint8_t*w=malloc((size_t)O*I),*s=malloc((size_t)(O/128)*(I/128));
+ float*x=malloc((size_t)I*sizeof(float)),*got=malloc((size_t)O*sizeof(float)),*ref=malloc((size_t)O*sizeof(float));
+ for(int o=0;o<O;o++)for(int i=0;i<I;i++)w[(size_t)o*I+i]=((o*3+i*5)&1)?0xb8:0x38;
+ for(int i=0;i<(O/128)*(I/128);i++)s[i]=127;
+ for(int i=0;i<I;i++)x[i]=((i%13)-6)/8.0f;
+ Dsv4CudaTensor*t=NULL;
+ if(!dsv4_cuda_upload_fp8(&t,w,s,O,I,dev)){fprintf(stderr,"fp8 upload failed\n");return 2;}
+ if(!dsv4_cuda_matvec(t,got,x)){fprintf(stderr,"matvec failed\n");return 3;}
+ ref_matvec(ref,w,s,O,I,x);
+ for(int o=0;o<O;o++)if(fabsf(ref[o]-got[o])>.05f*fmaxf(1,fabsf(ref[o]))){fprintf(stderr,"matvec mismatch row=%d ref=%g got=%g\n",o,ref[o],got[o]);return 4;}
+ dsv4_cuda_tensor_free(t);free(ref);free(got);free(x);free(s);free(w);
+ puts("decode[1/5] matvec vs CPU: OK");
+ }
+
+ /* --- part 2: single-token attention_window ------------------------------ */
+ {
+ enum { WR=8192 }; /* wa output rows */
+ uint8_t*qkvw=malloc((size_t)(R+K)*H),*qbs=malloc((size_t)Q*R),*kvw=malloc((size_t)K*H);
+ uint8_t*qkvs=malloc((size_t)((R+K+127)/128)*((H+127)/128)),*kv_s=malloc((size_t)((K+127)/128)*((H+127)/128));
+ uint8_t*qb_s=malloc((size_t)((Q+127)/128)*((R+127)/128));
+ uint8_t*waw=malloc((size_t)WR*H),*waw_s=malloc((size_t)((WR+127)/128)*((H+127)/128));
+ uint8_t*wbw=malloc((size_t)H*WR),*wb_s=malloc((size_t)((H+127)/128)*((WR+127)/128));
+ float*one=malloc((size_t)H*sizeof(float)),*kvone=malloc((size_t)K*sizeof(float)),*sinkv=calloc((size_t)HEADS,sizeof(float));
+ uint16_t*qnone=malloc((size_t)R*sizeof(uint16_t));
+ float*rope_cos=malloc((size_t)MAX*ROPE*sizeof(float)),*rope_sin=calloc((size_t)MAX*ROPE,sizeof(float));
+ float*x=malloc((size_t)H*sizeof(float)),*got=malloc((size_t)H*sizeof(float));
+ if(!qkvw||!qbs||!kvw||!qkvs||!kv_s||!qb_s||!waw||!waw_s||!wbw||!wb_s||!one||!kvone||!sinkv||!qnone||!rope_cos||!rope_sin||!x||!got)return 5;
+ memset(qkvw,0,(size_t)(R+K)*H);memset(kvw,0,(size_t)K*H);memset(qbs,0,(size_t)Q*R);
+ for(int i=0;i<R+K;i++)qkvw[(size_t)i*H+i%H]=0x38;
+ for(int i=0;i<K;i++)kvw[(size_t)i*H+i%H]=0x38;
+ for(int i=0;i<Q;i++)qbs[(size_t)i*R+i%R]=0x38;
+ memset(qkvs,127,(size_t)((R+K+127)/128)*((H+127)/128));
+ memset(kv_s,127,(size_t)((K+127)/128)*((H+127)/128));
+ memset(qb_s,127,(size_t)((Q+127)/128)*((R+127)/128));
+ memset(waw,0,(size_t)WR*H);
+ for(int i=0;i<WR*H;i+=((size_t)WR*H+1))waw[i]=0x38;
+ memset(waw_s,127,(size_t)((WR+127)/128)*((H+127)/128));
+ memset(wbw,0,(size_t)H*WR);
+ for(int i=0;i<H*WR;i+=((size_t)H*WR+1))wbw[i]=0x38;
+ memset(wb_s,127,(size_t)((H+127)/128)*((WR+127)/128));
+ for(int i=0;i<H;i++)one[i]=1;
+ for(int i=0;i<K;i++)kvone[i]=1;
+ for(int i=0;i<R;i++)qnone[i]=0x3f80;
+ for(int i=0;i<MAX*ROPE;i++)rope_cos[i]=1;
+ for(int i=0;i<H;i++)x[i]=(float)((i%7)-3)/8;
+
+ Dsv4CudaTensor *an=NULL,*qa=NULL,*qn=NULL,*qb=NULL,*kv=NULL,*kvn=NULL,*sink=NULL,*wa=NULL,*wb=NULL;
+ if(!dsv4_cuda_upload_f32(&an,one,H,1,dev)||!dsv4_cuda_upload_fp8(&qa,qkvw,qkvs,R+K,H,dev)||
+ !dsv4_cuda_upload_bf16(&qn,qnone,R,1,dev)||!dsv4_cuda_upload_fp8(&qb,qbs,qb_s,Q,R,dev)||
+ !dsv4_cuda_upload_fp8(&kv,kvw,kv_s,K,H,dev)||!dsv4_cuda_upload_f32(&kvn,kvone,K,1,dev)||
+ !dsv4_cuda_upload_f32(&sink,sinkv,HEADS,1,dev)||!dsv4_cuda_upload_fp8_bf16(&wa,waw,waw_s,WR,H,dev)||
+ !dsv4_cuda_upload_fp8(&wb,wbw,wb_s,H,WR,dev)){fprintf(stderr,"attention weight upload failed\n");return 6;}
+ Dsv4CudaKvCache*cache=dsv4_cuda_kv_create(dev,MAX,K,MAX,ROPE,rope_cos,rope_sin,rope_cos,rope_sin);
+ Dsv4CudaActivation*in=dsv4_cuda_activation_create(dev,H),*out=dsv4_cuda_activation_create(dev,H);
+ if(!cache||!in||!out){fprintf(stderr,"kv/activation alloc failed\n");return 7;}
+ if(!dsv4_cuda_activation_upload(in,x,H)){fprintf(stderr,"input upload failed\n");return 8;}
+ for(int t=0;t<T;t++){
+ if(!dsv4_cuda_decode_state_set(dev,t,t)||
+ !dsv4_cuda_attention_window(in,an,qa,qn,qb,kv,kvn,sink,wa,wb,NULL,NULL,NULL,NULL,0,HEADS,K,ROPE*2,GROUPS,t,1e-6f,cache,out)){
+ fprintf(stderr,"attention_window failed at token %d\n",t);return 9;
+ }
+ if(!dsv4_cuda_activation_download(got,out,H)||!dsv4_cuda_activation_sync(out)){fprintf(stderr,"output download failed\n");return 10;}
+ for(int i=0;i<H;i++)if(!isfinite(got[i])){fprintf(stderr,"non-finite attention out at %d:%d\n",t,i);return 11;}
+ }
+ dsv4_cuda_activation_free(out);dsv4_cuda_activation_free(in);dsv4_cuda_kv_free(cache);
+ dsv4_cuda_tensor_free(wb);dsv4_cuda_tensor_free(wa);dsv4_cuda_tensor_free(sink);dsv4_cuda_tensor_free(kvn);
+ dsv4_cuda_tensor_free(kv);dsv4_cuda_tensor_free(qb);dsv4_cuda_tensor_free(qn);dsv4_cuda_tensor_free(qa);dsv4_cuda_tensor_free(an);
+ free(got);free(x);free(rope_sin);free(rope_cos);free(qnone);free(sinkv);free(kvone);free(one);
+ free(wb_s);free(wbw);free(waw_s);free(waw);free(qb_s);free(qbs);free(kv_s);free(kvw);free(qkvs);free(qkvw);
+ puts("decode[2/5] attention_window: OK");
+ }
+
+ /* --- part 3: expert bank + routed MoE (single token) -------------------- */
+ {
+ enum { I=2048,E=256 };
+ /* The expert BANK stores FP4 (I*H/2 bytes), but the shared experts are
+ * uploaded as full FP8 (I*H bytes). Two different buffers. */
+ size_t w1=(size_t)I*H/2,s1=(size_t)I*H/32,w2=(size_t)H*I/2,s2=(size_t)H*I/32;
+ size_t sf_w=(size_t)I*H,sf_s=(size_t)((I+127)/128)*((H+127)/128),sd_s=(size_t)((H+127)/128)*((I+127)/128);
+ Dsv4CudaTensor*sg=NULL,*su=NULL,*sd=NULL,*gate=NULL;
+ uint8_t*w=calloc(w1,1),*s=malloc(s1),*dw=calloc(w2,1),*ds=malloc(s2);
+ uint8_t*sfw=calloc(sf_w,1),*sfs=malloc(sf_s),*sdfw=calloc(sf_w,1),*sdfs=malloc(sd_s);
+ float*gw_router=calloc((size_t)E*H,sizeof(float));
+ if(!w||!s||!dw||!ds||!sfw||!sfs||!sdfw||!sdfs||!gw_router)return 12;
+ memset(s,127,s1);memset(ds,127,s2);memset(sfs,127,sf_s);memset(sdfs,127,sd_s);
+ if(!dsv4_cuda_upload_fp8(&sg,sfw,sfs,I,H,dev)){fprintf(stderr,"sg upload failed\n");return 13;}
+ if(!dsv4_cuda_upload_fp8(&su,sfw,sfs,I,H,dev)){fprintf(stderr,"su upload failed\n");return 13;}
+ if(!dsv4_cuda_upload_fp8(&sd,sdfw,sdfs,H,I,dev)){fprintf(stderr,"sd upload failed\n");return 13;}
+ if(!dsv4_cuda_upload_f32(&gate,gw_router,E,H,dev)){fprintf(stderr,"gate upload failed\n");return 13;}
+ Dsv4CudaExpertSet*bank=dsv4_cuda_expert_bank_create(E,H,I,dev,sg,su,sd);
+ if(!bank){fprintf(stderr,"expert bank create failed\n");return 14;}
+ for(int e=0;e<E;e++){Dsv4CudaTensor*g=NULL,*u=NULL,*d=NULL;
+ if(!dsv4_cuda_expert_bank_upload(bank,e,w,s,w,s,dw,ds,&g,&u,&d)){fprintf(stderr,"bank upload %d failed\n",e);return 15;}
+ dsv4_cuda_tensor_free(g);dsv4_cuda_tensor_free(u);dsv4_cuda_tensor_free(d);}
+ float*x=malloc((size_t)H*sizeof(float));
+ Dsv4CudaActivation*in=dsv4_cuda_activation_create(dev,H),*out=dsv4_cuda_activation_create(dev,H);
+ float*got=malloc((size_t)H*sizeof(float));
+ if(!x||!in||!out||!got)return 16;
+ for(int i=0;i<H;i++)x[i]=(float)((i%11)-5)/16;
+ if(!dsv4_cuda_activation_upload(in,x,H)){fprintf(stderr,"moe input upload failed\n");return 17;}
+ if(!dsv4_cuda_route_moe(in,gate,NULL,0,1.f,bank,7.f,out)){fprintf(stderr,"route_moe failed\n");return 18;}
+ if(!dsv4_cuda_activation_download(got,out,H)||!dsv4_cuda_activation_sync(out)){fprintf(stderr,"moe download failed\n");return 19;}
+ for(int i=0;i<H;i++)if(!isfinite(got[i])){fprintf(stderr,"non-finite moe out at %d\n",i);return 20;}
+ dsv4_cuda_activation_free(out);dsv4_cuda_activation_free(in);free(got);free(x);
+ dsv4_cuda_expert_set_free(bank);dsv4_cuda_tensor_free(gate);dsv4_cuda_tensor_free(sd);dsv4_cuda_tensor_free(su);dsv4_cuda_tensor_free(sg);
+ free(gw_router);free(ds);free(dw);free(s);free(w);free(sdfs);free(sdfw);free(sfs);free(sfw);
+ puts("decode[3/5] expert bank + route_moe: OK");
+ }
+
+ /* --- part 4: mHC pre/post + rmsnorm ------------------------------------- */
+ {
+ enum { M=4,N=24,parts=8 };
+ long long MH=(long long)M*H;
+ Dsv4CudaTensor *fn=NULL,*scale=NULL,*base=NULL,*norm=NULL;
+ float*fnv=malloc((size_t)N*MH*sizeof(float)),*scv=malloc(3*sizeof(float)),*bsv=malloc((size_t)N*sizeof(float)),*nmv=malloc((size_t)H*sizeof(float));
+ float*x=malloc((size_t)MH*sizeof(float)),*got=malloc((size_t)MH*sizeof(float));
+ if(!fnv||!scv||!bsv||!nmv||!x||!got)return 21;
+ for(int i=0;i<N*MH;i++)fnv[i]=0.01f;scv[0]=1;scv[1]=1;scv[2]=1;for(int i=0;i<N;i++)bsv[i]=0;for(int i=0;i<H;i++)nmv[i]=1;
+ for(int i=0;i<MH;i++)x[i]=(float)((i%5)-2)/4;
+ if(!dsv4_cuda_upload_f32(&fn,fnv,N,MH,dev)||!dsv4_cuda_upload_f32(&scale,scv,3,1,dev)||
+ !dsv4_cuda_upload_f32(&base,bsv,N,1,dev)||!dsv4_cuda_upload_f32(&norm,nmv,H,1,dev)){fprintf(stderr,"mhc weight upload failed\n");return 22;}
+ Dsv4CudaActivation*r=dsv4_cuda_activation_create(dev,MH),*state=dsv4_cuda_activation_create(dev,32),*in=dsv4_cuda_activation_create(dev,H),*out=dsv4_cuda_activation_create(dev,MH);
+ if(!r||!state||!in||!out)return 23;
+ if(!dsv4_cuda_activation_upload(r,x,MH)||!dsv4_cuda_mhc_pre_norm(r,fn,scale,base,norm,M,H,1e-6f,1e-6f,1e-6f,1.f,1,1e-6f,state,in)||
+ !dsv4_cuda_mhc_post(in,r,state,M,H,out)||!dsv4_cuda_activation_download(got,out,MH)||!dsv4_cuda_activation_sync(out)){fprintf(stderr,"mhc run failed\n");return 24;}
+ for(int i=0;i<MH;i++)if(!isfinite(got[i])){fprintf(stderr,"non-finite mhc out at %d\n",i);return 25;}
+ dsv4_cuda_activation_free(out);dsv4_cuda_activation_free(in);dsv4_cuda_activation_free(state);dsv4_cuda_activation_free(r);
+ dsv4_cuda_tensor_free(norm);dsv4_cuda_tensor_free(base);dsv4_cuda_tensor_free(scale);dsv4_cuda_tensor_free(fn);
+ free(got);free(x);free(nmv);free(bsv);free(scv);free(fnv);
+ puts("decode[4/5] mHC pre/post + rmsnorm: OK");
+ }
+
+ /* --- part 5: router gate f32 upload + tensor_device + dsv4_cuda_route --- */
+ {
+ enum { E=256, DH=64 };
+ float*gw=malloc((size_t)E*DH*sizeof(float)),*bv=calloc((size_t)E,sizeof(float));
+ float*x=malloc((size_t)DH*sizeof(float));
+ if(!gw||!bv||!x)return 26;
+ for(int e=0;e<E;e++)gw[(size_t)e*DH+0]=(float)e+1.0f; /* expert e wins on const input */
+ for(int i=0;i<DH;i++)x[i]= i==0?1.0f:0.0f;
+ Dsv4CudaTensor*gate=NULL,*bias=NULL;
+ if(!dsv4_cuda_upload_f32(&gate,gw,E,DH,dev)||!dsv4_cuda_upload_f32(&bias,bv,E,1,dev)){fprintf(stderr,"router upload failed\n");return 27;}
+ if(dsv4_cuda_tensor_device(gate)!=dev){fprintf(stderr,"tensor_device=%d expected %d\n",dsv4_cuda_tensor_device(gate),dev);return 28;}
+ if(dsv4_cuda_tensor_bytes(gate)!=(long long)E*DH*sizeof(float)){fprintf(stderr,"tensor_bytes=%lld\n",dsv4_cuda_tensor_bytes(gate));return 29;}
+ Dsv4CudaActivation*act=dsv4_cuda_activation_create(dev,DH);
+ if(!act||!dsv4_cuda_activation_upload(act,x,DH)){fprintf(stderr,"router activation failed\n");return 30;}
+ int ids[6]={0};float wts[6]={0};
+ static const int forced[6]={7,8,9,10,11,12};
+ if(!dsv4_cuda_route(act,gate,bias,forced,1.0f,ids,wts)){fprintf(stderr,"route failed\n");return 31;}
+ for(int k=0;k<6;k++)if(ids[k]!=forced[k]||!isfinite(wts[k])){fprintf(stderr,"route mismatch k=%d id=%d\n",k,ids[k]);return 32;}
+ dsv4_cuda_activation_free(act);dsv4_cuda_tensor_free(bias);dsv4_cuda_tensor_free(gate);
+ free(x);free(bv);free(gw);
+ puts("decode[5/5] router gate upload + tensor_device + route: OK");
+ }
+
+ /* --- part 6: fp4 expert matvec against the engine CPU reference -------- */
+ {
+ enum { O=512, I=4096 };
+ /* ExpertStore layout: packed nibbles (even column in low nibble),
+ * per-32-column UE8M0 scale, one scale group per row. Same layout the
+ * engine's matmul_mxfp4 reference consumes and the mirror attach
+ * uploads. */
+ size_t pw=(size_t)O*I/2, ps=(size_t)O*(I/32);
+ uint8_t*w=malloc(pw),*s=malloc(ps);
+ float*x=malloc((size_t)I*sizeof(float)),*got=malloc((size_t)O*sizeof(float)),*ref=malloc((size_t)O*sizeof(float));
+ if(!w||!s||!x||!got||!ref)return 33;
+ for(size_t i=0;i<pw;i++)w[i]=(uint8_t)(rand()&0xff);
+ for(size_t i=0;i<ps;i++)s[i]=(uint8_t)(120+(rand()%8)); /* e8m0 2^-7..1 */
+ for(int i=0;i<I;i++)x[i]=((i%17)-8)/8.0f;
+ for(int o=0;o<O;o++){
+ const uint8_t*wr=w+(size_t)o*(I/2),*sr=s+(size_t)o*(I/32);
+ double acc=0;
+ for(int g=0;g<I/32;g++){
+ float sc=e8m0_decode(sr[g]),ga=0;
+ for(int i=g*32;i<g*32+32;i+=2){
+ uint8_t byte=wr[i>>1];
+ ga+=x[i]*fp4_decode(byte&0xF);
+ ga+=x[i+1]*fp4_decode(byte>>4);
+ }
+ acc+=(double)ga*sc;
+ }
+ ref[o]=(float)acc;
+ }
+ Dsv4CudaTensor*t=NULL;
+ if(!dsv4_cuda_upload_fp4(&t,w,s,O,I,dev)){fprintf(stderr,"fp4 upload failed\n");return 34;}
+ if(!dsv4_cuda_matvec(t,got,x)){fprintf(stderr,"fp4 matvec failed\n");return 35;}
+ for(int o=0;o<O;o++)if(fabsf(ref[o]-got[o])>.05f*fmaxf(1,fabsf(ref[o]))){fprintf(stderr,"fp4 matvec mismatch row=%d ref=%g got=%g\n",o,ref[o],got[o]);return 36;}
+ dsv4_cuda_tensor_free(t);free(ref);free(got);free(x);free(s);free(w);
+ puts("decode[6/6] fp4 expert matvec vs CPU: OK");
+ }
+
+ dsv4_cuda_shutdown();
+ puts("dsv4 decode path through loader: OK");
+ return 0;
+}
+
+
diff --git a/c/tests/test_dsv4_gpu_grouped.c b/c/tests/test_dsv4_gpu_grouped.c
new file mode 100644
index 00000000..b1a35137
--- /dev/null
+++ b/c/tests/test_dsv4_gpu_grouped.c
@@ -0,0 +1,82 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include "backend_cuda_dsv4.h"
+
+static float e4m3_decode(unsigned char v) {
+ int sign = v >> 7, e = (v >> 3) & 15, m = v & 7;
+ float val;
+ if (!e) val = ldexpf((float)m, -9);
+ else if (e == 15) val = m == 7 ? NAN : ldexpf(1.0f + m / 8.0f, 8);
+ else val = ldexpf(1.0f + m / 8.0f, e - 7);
+ return sign ? -val : val;
+}
+static unsigned char e4m3_encode(float x) {
+ float best = 0, bd = 1e30f;
+ unsigned char bc = 0;
+ for (int v = 0; v < 255; v++) {
+ float z = e4m3_decode((unsigned char)v);
+ if (isnan(z)) continue;
+ float d = fabsf(z - x);
+ if (d < bd || (d == bd && !(v & 1) && (bc & 1))) { bd = d; bc = (unsigned char)v; }
+ }
+ return bc;
+}
+static float e8m0(unsigned char v) {
+ return v == 255 ? NAN : ldexpf(1.0f, (int)v - 127);
+}
+
+int main(void) {
+ int device = 0;
+ if (!dsv4_cuda_init(&device, 1)) { fprintf(stderr, "no device\n"); return 2; }
+ const int groups = 4, o_rank = 128, group_width = 256;
+ const int O = groups * o_rank, I = group_width;
+ unsigned char *w = malloc((size_t)O * I);
+ unsigned char *scales = malloc((size_t)(O / 128) * (I / 128));
+ float *x = malloc((size_t)groups * I * sizeof(float));
+ float *cpu = malloc((size_t)O * sizeof(float));
+ float *gpu = malloc((size_t)O * sizeof(float));
+ if (!w || !scales || !x || !cpu || !gpu) return 6;
+ for (int i = 0; i < O * I; i++)
+ w[i] = e4m3_encode((float)((i % 97) - 48) / 64.0f);
+ for (int b = 0; b < (O / 128) * (I / 128); b++)
+ scales[b] = (unsigned char)((b % 7) - 3 + 127);
+ for (int i = 0; i < groups * I; i++)
+ x[i] = (float)((i % 61) - 30) / 16.0f;
+ int scale_rows_per_group = (o_rank + 127) / 128;
+ int scale_columns = (group_width + 127) / 128;
+ for (int g = 0; g < groups; g++)
+ for (int o = 0; o < o_rank; o++) {
+ float s = 0;
+ for (int i = 0; i < I; i++) {
+ int b128 = (size_t)g * scale_rows_per_group * scale_columns +
+ (o / 128) * scale_columns + i / 128;
+ float wv = e4m3_decode(w[((size_t)g * o_rank + o) * I + i]) *
+ e8m0(scales[b128]);
+ s += x[(size_t)g * I + i] * wv;
+ }
+ cpu[(size_t)g * o_rank + o] = s;
+ }
+ Dsv4CudaTensor *t = NULL;
+ if (!dsv4_cuda_upload_fp8(&t, w, scales, O, I, device)) {
+ fprintf(stderr, "upload fail\n"); return 3;
+ }
+ if (!dsv4_cuda_matvec_grouped(t, gpu, x, groups)) {
+ fprintf(stderr, "grouped fail\n"); return 4;
+ }
+ double worst = 0;
+ for (int i = 0; i < O; i++) {
+ double d = fabsf(gpu[i] - cpu[i]);
+ if (d > worst) worst = d;
+ if (fabsf(gpu[i] - cpu[i]) > 1e-2f) {
+ fprintf(stderr, "mismatch at %d gpu=%f cpu=%f\n", i, gpu[i], cpu[i]);
+ return 5;
+ }
+ }
+ printf("grouped matvec: OK worst=%.2e (O=%d I=%d groups=%d)\n",
+ worst, O, I, groups);
+ dsv4_cuda_tensor_free(t);
+ dsv4_cuda_shutdown();
+ return 0;
+}
diff --git a/c/tests/test_dsv4_sparse_attn_batch_cuda.c b/c/tests/test_dsv4_sparse_attn_batch_cuda.c
new file mode 100644
index 00000000..f2d4c22b
--- /dev/null
+++ b/c/tests/test_dsv4_sparse_attn_batch_cuda.c
@@ -0,0 +1,254 @@
+/* Validates dsv4_cuda_sparse_attn_batch and dsv4_cuda_matmul_bf16_batch
+ * against inline CPU references that mirror coli_v4_sparse_attention_ref and
+ * the compressor's bf16 projection loop. Build with cl against
+ * coli_cuda_dsv4_dg.lib. */
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "../backend_cuda_dsv4.h"
+
+static float bf16_round(float x) {
+ uint32_t bits;
+ memcpy(&bits, &x, 4);
+ uint32_t lsb = (bits >> 16) & 1;
+ bits += 0x7fffu + lsb;
+ bits &= 0xffff0000u;
+ float out;
+ memcpy(&out, &bits, 4);
+ return out;
+}
+
+static float bf16_decode(uint16_t h) {
+ uint32_t bits = (uint32_t)h << 16;
+ float out;
+ memcpy(&out, &bits, 4);
+ return out;
+}
+
+static float frand(unsigned *seed) {
+ *seed = *seed * 1664525u + 1013904223u;
+ return ((*seed >> 8) & 0xffff) / 65536.0f - 0.5f;
+}
+
+/* CPU reference: same contract as the GPU kernel (window range + compressed
+ * prefix over a linear slab), same math as coli_v4_sparse_attention_ref. */
+static void ref_token(float *out, const float *q, const float *vals,
+ const float *sinks, int wo, int wn, int cn,
+ int comp_base, int heads, int dim, float scale) {
+ float *scores = malloc((size_t)(wn + cn) * sizeof(*scores));
+ for (int h = 0; h < heads; h++) {
+ const float *qh = q + (size_t)h * dim;
+ float maximum = -INFINITY;
+ for (int r = 0; r < wn + cn; r++) {
+ const float *v = vals + (size_t)(r < wn ? wo + r
+ : comp_base + r - wn) * dim;
+ float s = 0.0f;
+ for (int d = 0; d < dim; d++) s += qh[d] * v[d];
+ s *= scale;
+ scores[r] = s;
+ if (s > maximum) maximum = s;
+ }
+ float den = expf(sinks[h] - maximum);
+ float *oh = out + (size_t)h * dim;
+ memset(oh, 0, (size_t)dim * sizeof(*oh));
+ for (int r = 0; r < wn + cn; r++) {
+ float p = expf(scores[r] - maximum);
+ den += p;
+ p = bf16_round(p);
+ const float *v = vals + (size_t)(r < wn ? wo + r
+ : comp_base + r - wn) * dim;
+ for (int d = 0; d < dim; d++) oh[d] += p * v[d];
+ }
+ for (int d = 0; d < dim; d++) oh[d] = bf16_round(oh[d] / den);
+ }
+ free(scores);
+}
+
+int main(void) {
+ enum { HEADS = 64, DIM = 512, WINDOW = 128, T = 64, START = 200,
+ COMP = 60 };
+ int dev = 0;
+ if (!dsv4_cuda_init(&dev, 1)) return 1;
+ unsigned seed = 12345;
+
+ /* ---- sparse attention ---- */
+ int first = START - WINDOW + 1;
+ int pre = START - first;
+ int value_rows = pre + T + COMP;
+ int comp_base = pre + T;
+ float *q = malloc((size_t)T * HEADS * DIM * sizeof(float));
+ float *vals = malloc((size_t)value_rows * DIM * sizeof(float));
+ float *sinks = malloc(HEADS * sizeof(float));
+ int *meta = malloc((size_t)T * 3 * sizeof(int));
+ float *got = malloc((size_t)T * HEADS * DIM * sizeof(float));
+ float *want = malloc((size_t)HEADS * DIM * sizeof(float));
+ if (!q || !vals || !sinks || !meta || !got || !want) return 2;
+ for (size_t i = 0; i < (size_t)T * HEADS * DIM; i++) q[i] = frand(&seed);
+ for (size_t i = 0; i < (size_t)value_rows * DIM; i++) vals[i] = frand(&seed);
+ for (int h = 0; h < HEADS; h++) sinks[h] = frand(&seed) * 4.0f;
+ for (int t = 0; t < T; t++) {
+ int pos = START + t;
+ int wfirst = pos - WINDOW + 1;
+ if (wfirst < 0) wfirst = 0;
+ meta[3 * t] = wfirst - first;
+ meta[3 * t + 1] = pos - wfirst + 1;
+ meta[3 * t + 2] = COMP * (t + 1) / T; /* growing prefix */
+ }
+ float scale = 1.0f / sqrtf((float)DIM);
+ if (!dsv4_cuda_sparse_attn_batch(dev, q, vals, sinks, meta, value_rows,
+ comp_base, HEADS, DIM, T, scale, got)) {
+ fprintf(stderr, "sparse_attn_batch refused\n");
+ return 3;
+ }
+ double worst = 0.0;
+ for (int t = 0; t < T; t++) {
+ ref_token(want, q + (size_t)t * HEADS * DIM, vals, sinks,
+ meta[3 * t], meta[3 * t + 1], meta[3 * t + 2],
+ comp_base, HEADS, DIM, scale);
+ for (size_t i = 0; i < (size_t)HEADS * DIM; i++) {
+ double diff = fabs((double)want[i] -
+ got[(size_t)t * HEADS * DIM + i]);
+ if (diff > worst) worst = diff;
+ if (diff > 2e-3) {
+ fprintf(stderr, "attention mismatch t=%d i=%zu want=%g got=%g\n",
+ t, i, want[i], got[(size_t)t * HEADS * DIM + i]);
+ return 4;
+ }
+ }
+ }
+ printf("sparse attention batch: max |diff| = %g\n", worst);
+
+ /* ---- cached sparse attention (ring + separate chunk buffer) ----
+ * Only the historic rows go into the ring; the chunk rows ride along as
+ * their own buffer exactly like the engine passes them (appending the
+ * chunk first would alias the oldest window rows once positions exceed
+ * the window). */
+ enum { LAYER = 7, TOPK = 24 };
+ if (!dsv4_cuda_kv_ring_append(dev, LAYER, vals, first, pre, WINDOW,
+ DIM) ||
+ !dsv4_cuda_kv_comp_append(dev, LAYER, vals + (size_t)comp_base * DIM,
+ 0, COMP, DIM)) {
+ fprintf(stderr, "kv cache seed refused\n");
+ return 9;
+ }
+ if (!dsv4_cuda_sparse_attn_batch_cached(dev, LAYER, q,
+ vals + (size_t)pre * DIM, START,
+ sinks, meta, first, COMP, HEADS,
+ DIM, T, scale, got)) {
+ fprintf(stderr, "sparse_attn_batch_cached refused\n");
+ return 13;
+ }
+ worst = 0.0;
+ for (int t = 0; t < T; t++) {
+ ref_token(want, q + (size_t)t * HEADS * DIM, vals, sinks,
+ meta[3 * t], meta[3 * t + 1], meta[3 * t + 2],
+ comp_base, HEADS, DIM, scale);
+ for (size_t i = 0; i < (size_t)HEADS * DIM; i++) {
+ double diff = fabs((double)want[i] -
+ got[(size_t)t * HEADS * DIM + i]);
+ if (diff > worst) worst = diff;
+ if (diff > 2e-3) {
+ fprintf(stderr, "cached attention mismatch t=%d i=%zu want=%g got=%g\n",
+ t, i, want[i], got[(size_t)t * HEADS * DIM + i]);
+ return 14;
+ }
+ }
+ }
+ printf("cached sparse attention: max |diff| = %g\n", worst);
+
+ /* ---- cached indexed sparse attention (subset selection) ---- */
+ int *sel = malloc((size_t)T * TOPK * sizeof(*sel));
+ int *meta_idx = malloc((size_t)T * 3 * sizeof(*meta_idx));
+ int *pool = malloc((size_t)COMP * sizeof(*pool));
+ float *vals_g = malloc((size_t)(WINDOW + TOPK) * DIM * sizeof(*vals_g));
+ if (!sel || !meta_idx || !pool || !vals_g) return 10;
+ for (int t = 0; t < T; t++) {
+ int cn_full = meta[3 * t + 2];
+ int sn = cn_full < TOPK ? cn_full : TOPK;
+ meta_idx[3 * t] = meta[3 * t];
+ meta_idx[3 * t + 1] = meta[3 * t + 1];
+ meta_idx[3 * t + 2] = sn;
+ for (int i = 0; i < cn_full; i++) pool[i] = i;
+ for (int i = 0; i < sn; i++) { /* partial shuffle: unsorted subset */
+ int j = i + (int)((frand(&seed) + 0.5f) * (cn_full - i));
+ if (j >= cn_full) j = cn_full - 1;
+ int tmp = pool[i]; pool[i] = pool[j]; pool[j] = tmp;
+ sel[(size_t)t * TOPK + i] = pool[i];
+ }
+ }
+ if (!dsv4_cuda_sparse_attn_batch_cached_idx(dev, LAYER, q,
+ vals + (size_t)pre * DIM, START,
+ sinks, meta_idx,
+ sel, TOPK, first, COMP, HEADS,
+ DIM, T, scale, got)) {
+ fprintf(stderr, "sparse_attn_batch_cached_idx refused\n");
+ return 11;
+ }
+ worst = 0.0;
+ for (int t = 0; t < T; t++) {
+ int wo = meta_idx[3 * t], wn = meta_idx[3 * t + 1],
+ sn = meta_idx[3 * t + 2];
+ memcpy(vals_g, vals + (size_t)wo * DIM,
+ (size_t)wn * DIM * sizeof(*vals_g));
+ for (int i = 0; i < sn; i++)
+ memcpy(vals_g + (size_t)(wn + i) * DIM,
+ vals + (size_t)(comp_base + sel[(size_t)t * TOPK + i]) * DIM,
+ (size_t)DIM * sizeof(*vals_g));
+ ref_token(want, q + (size_t)t * HEADS * DIM, vals_g, sinks,
+ 0, wn, sn, wn, HEADS, DIM, scale);
+ for (size_t i = 0; i < (size_t)HEADS * DIM; i++) {
+ double diff = fabs((double)want[i] -
+ got[(size_t)t * HEADS * DIM + i]);
+ if (diff > worst) worst = diff;
+ if (diff > 2e-3) {
+ fprintf(stderr, "idx attention mismatch t=%d i=%zu want=%g got=%g\n",
+ t, i, want[i], got[(size_t)t * HEADS * DIM + i]);
+ return 12;
+ }
+ }
+ }
+ printf("cached idx sparse attention: max |diff| = %g\n", worst);
+ free(sel); free(meta_idx); free(pool); free(vals_g);
+
+ /* ---- bf16 batch matmul ---- */
+ enum { O = 256, I = 4096, TT = 64 };
+ uint16_t *w = malloc((size_t)O * I * sizeof(*w));
+ float *x = malloc((size_t)TT * I * sizeof(float));
+ float *y = malloc((size_t)TT * O * sizeof(float));
+ if (!w || !x || !y) return 5;
+ for (size_t i = 0; i < (size_t)O * I; i++) {
+ float v = frand(&seed);
+ uint32_t bits;
+ memcpy(&bits, &v, 4);
+ w[i] = (uint16_t)(bits >> 16);
+ }
+ for (size_t i = 0; i < (size_t)TT * I; i++) x[i] = frand(&seed);
+ Dsv4CudaTensor *t = NULL;
+ if (!dsv4_cuda_upload_bf16(&t, w, O, I, dev)) return 6;
+ if (!dsv4_cuda_matmul_bf16_batch(t, x, TT, y)) {
+ fprintf(stderr, "matmul_bf16_batch refused\n");
+ return 7;
+ }
+ worst = 0.0;
+ for (int tt = 0; tt < TT; tt++)
+ for (int o = 0; o < O; o++) {
+ float sum = 0.0f;
+ for (int i = 0; i < I; i++)
+ sum += bf16_decode(w[(size_t)o * I + i]) * x[(size_t)tt * I + i];
+ double diff = fabs((double)sum - y[(size_t)tt * O + o]);
+ double mag = fabs((double)sum) + 1.0;
+ if (diff / mag > worst) worst = diff / mag;
+ if (diff / mag > 1e-4) {
+ fprintf(stderr, "bf16 matmul mismatch t=%d o=%d want=%g got=%g\n",
+ tt, o, sum, y[(size_t)tt * O + o]);
+ return 8;
+ }
+ }
+ printf("bf16 batch matmul: max rel diff = %g\n", worst);
+ dsv4_cuda_tensor_free(t);
+ dsv4_cuda_shutdown();
+ puts("dsv4 sparse attn batch: OK");
+ return 0;
+}