Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
Remove vendored DeepSeek V4 CUDA paths
ZacharyZcR763dac7882da
diff --git a/c/Makefile b/c/Makefileindex 747f63b9..d2eb7423 100644--- a/c/Makefile+++ b/c/Makefile@@ -175,7 +175,6 @@ CUDA_DLL ?= 0COLI_ANS ?= 0DIETGPU_ROOT ?=DEEPGEMM ?= 0-VLLM_MHC ?= 0NCCL ?= 0ifneq ($(IS_WIN),)# the CUDA installer sets CUDA_PATH system-wide (e.g. C:\Program Files\NVIDIA@@ -437,23 +436,10 @@ endif# objects are self-contained: backend_cuda_dsv4.* include only their own header# and the CUDA runtime, so they build and their tests run without the engine.#-# VLLM_MHC, DEEPGEMM and FLASHINFER below are UNSUPPORTED. They are eight build-# configurations and CI compiles none of them: each needs an external checkout-# (TILELANG_HOME, DEEPGEMM_HOME, FLASHINFER_HOME) and DeepGEMM's kernels want-# Blackwell to run. That is a statement about verification, not quality -- an-# untested build path rots silently, and whoever is here in six months inherits-# it. They gate nothing and block nothing: the default build (all three off) is-# the one CI compiles and the one to report bugs against. Enable them only if-# you have the external trees and are prepared to debug the combination.+# DEEPGEMM is unsupported: CI cannot compile it without an external checkout,+# and its kernels require Blackwell to run. It gates nothing and blocks nothing;+# the default build is the supported configuration.DSV4_CUDA_OBJ = backend_cuda_dsv4.o-ifeq ($(VLLM_MHC),1)-ifndef TILELANG_HOME-$(error TILELANG_HOME must point to an installed TileLang package)-endif-CFLAGS += -DCOLI_DSV4_VLLM_MHC-override NVCCFLAGS += -DCOLI_DSV4_VLLM_MHC-DSV4_CUDA_OBJ += backend_cuda_dsv4_mhc_vllm.o-endififeq ($(DEEPGEMM),1)ifndef DEEPGEMM_HOME$(error DEEPGEMM_HOME must point to a DeepGEMM include tree)@@ -469,14 +455,6 @@ override NVCC_STD := c++20override NVCCFLAGS += --expt-relaxed-constexpr --expt-extended-lambda -DCOLI_DSV4_DEEPGEMM -I"$(DEEPGEMM_HOME)"LDFLAGS += -lcudaendif-ifeq ($(FLASHINFER),1)-ifndef FLASHINFER_HOME-$(error FLASHINFER_HOME must point to a FlashInfer source checkout)-endif-CFLAGS += -DCOLI_DSV4_FLASHINFER-override NVCCFLAGS += -DCOLI_DSV4_FLASHINFER-DSV4_CUDA_OBJ += backend_cuda_dsv4_flashinfer.o backend_cuda_dsv4_sparse_mla_flashinfer.o backend_cuda_dsv4_qkv_vllm.o-endififeq ($(NCCL),1)CFLAGS += -DCOLI_DSV4_NCCLoverride NVCCFLAGS += -DCOLI_DSV4_NCCL@@ -687,32 +665,9 @@ backend_cuda.o: backend_cuda.cu backend_cuda.h backend_gpu_compat.h .build-confi"$(GPUCC)" $(GPUFLAGS) -c backend_cuda.cu -o $@-backend_cuda_dsv4.o: backend_cuda_dsv4.cu backend_cuda_dsv4.h backend_cuda_dsv4_flashinfer.h .build-config+backend_cuda_dsv4.o: backend_cuda_dsv4.cu backend_cuda_dsv4.h .build-config@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }"$(NVCC)" $(NVCCFLAGS) -c backend_cuda_dsv4.cu -o $@--backend_cuda_dsv4_mhc_vllm.o: backend_cuda_dsv4_mhc_vllm.cu backend_cuda_dsv4_mhc_vllm.h .build-config- @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }- "$(NVCC)" -O3 -std=c++20 -arch=sm_120 -DENABLE_BF16 \- -I"$(TILELANG_HOME)/src" -I"$(TILELANG_HOME)/3rdparty/cutlass/include" \- -c backend_cuda_dsv4_mhc_vllm.cu -o $@--backend_cuda_dsv4_flashinfer.o: backend_cuda_dsv4_flashinfer.cu backend_cuda_dsv4_flashinfer.h .build-config- @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }- "$(NVCC)" -O3 -std=c++17 -arch=sm_120a --expt-relaxed-constexpr -diag-suppress=20012 -diag-suppress=20013 -diag-suppress=20015 \- -I"$(FLASHINFER_HOME)/include" -I"$(FLASHINFER_HOME)/3rdparty/cutlass/include" \- -I"$(FLASHINFER_HOME)/3rdparty/cutlass/tools/util/include" -c backend_cuda_dsv4_flashinfer.cu -o $@--backend_cuda_dsv4_sparse_mla_flashinfer.o: $(FLASHINFER_HOME)/csrc/sparse_mla_sm120_decode_dsv4.cu .build-config- @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }- "$(NVCC)" -O3 -std=c++17 -gencode=arch=compute_120f,code=sm_120f --expt-relaxed-constexpr \- -I"$(FLASHINFER_HOME)/include" -I"$(FLASHINFER_HOME)/3rdparty/cutlass/include" \- -c "$<" -o $@--backend_cuda_dsv4_qkv_vllm.o: backend_cuda_dsv4_qkv_vllm.cu backend_cuda_dsv4_qkv_vllm.h .build-config- @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }- "$(NVCC)" -O3 -std=c++17 -gencode=arch=compute_120f,code=sm_120f \- --expt-relaxed-constexpr --expt-extended-lambda -c "$<" -o $@backend_metal.o: backend_metal.mm backend_metal.h$(METALXX) -c backend_metal.mm -o $@diff --git a/c/THIRD_PARTY_NOTICES.md b/c/THIRD_PARTY_NOTICES.mddeleted file mode 100644index dde8a3c6..00000000--- a/c/THIRD_PARTY_NOTICES.md+++ /dev/null@@ -1,60 +0,0 @@-# Third-party notices--Portions of `backend_cuda_dsv4.cu`, including the DeepSeek-V4 GPU router-selection algorithm, are adapted from `ds4_cuda.cu` in the ds4 project:-https://github.com/antirez/ds4--MIT License--Copyright (c) 2026 The ds4.c authors-Copyright (c) 2023-2026 The ggml authors--Permission is hereby granted, free of charge, to any person obtaining a copy-of this software and associated documentation files (the "Software"), to deal-in the Software without restriction, including without limitation the rights-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell-copies of the Software, and to permit persons to whom the Software is-furnished to do so, subject to the following conditions:--The above copyright notice and this permission notice shall be included in all-copies or substantial portions of the Software.--THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE-SOFTWARE.--`tests/bench_dsv4_flashinfer.cu` instantiates the FlashInfer SM120 grouped-MXFP4 GEMM interface from https://github.com/flashinfer-ai/flashinfer.--Copyright (c) 2023-2026 FlashInfer authors--Licensed under the Apache License, Version 2.0. You may obtain a copy at-https://www.apache.org/licenses/LICENSE-2.0.--The DeepSeek-V4 DeepGEMM oracle and benchmark tools follow the FP8 packing-and kernel-dispatch interfaces used by the vLLM project:-https://github.com/vllm-project/vllm--The pinned DeepSeek-V4 reference runner in `tools/dsv4_vllm_reference.py`-executes vLLM commit `ffd46bfab2128bb84146050e98b51a617c6575ab`-without replacing or modifying its kernels. It is retained as the behavioral-and performance oracle for the native Colibri port.--`backend_cuda_dsv4_mhc_vllm.cu` is the CUDA source generated from that exact-commit's TileLang mHC decode kernels for the DeepSeek-V4-Flash configuration-(`hc=4`, `hidden=4096`, `n_out=24`, `split_k=8`). It is kept as the pinned-upstream source for the native ABI integration.--`backend_cuda_dsv4_qkv_vllm.cu` adapts that commit's-`fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu`. The Torch registration-layer is replaced by a native C ABI; the CUDA Q RMSNorm, RoPE, UE8M0 FP8-quantization, and paged KV-cache insertion algorithm remains upstream code.--Copyright contributors to the vLLM project--Licensed under the Apache License, Version 2.0. You may obtain a copy at-https://www.apache.org/licenses/LICENSE-2.0.diff --git a/c/backend_cuda_dsv4.cu b/c/backend_cuda_dsv4.cuindex ce924944..d5bfc52b 100644--- a/c/backend_cuda_dsv4.cu+++ b/c/backend_cuda_dsv4.cu@@ -11,34 +11,24 @@#ifdef COLI_DSV4_NCCL#include <nccl.h>#endif-#ifdef COLI_DSV4_VLLM_MHC-#include "backend_cuda_dsv4_mhc_vllm.h"-#endif#ifdef COLI_DSV4_DEEPGEMM#include <cuda.h>#include <deep_gemm/impls/sm120_fp8_fp4_gemm_1d1d.cuh>#include <deep_gemm/impls/sm120_split_k_reduce.cuh>#include <deep_gemm/impls/sm120_bf16_gemm.cuh>#endif-#ifdef COLI_DSV4_FLASHINFER-#include "backend_cuda_dsv4_flashinfer.h"-#include "backend_cuda_dsv4_qkv_vllm.h"-#endif-struct Dsv4CudaTensor { void *w,*bf16_cache; uint8_t *scale,*fi_scale;int *dg_scale; int O,I,fmt,device,own_w,own_scale,own_dg_scale; long long bytes; };+struct Dsv4CudaTensor { void *w,*bf16_cache; uint8_t *scale;int *dg_scale; int O,I,fmt,device,own_w,own_scale,own_dg_scale; long long bytes; };struct Dsv4CudaActivation { float *data; long long elements; int device; };struct Dsv4CudaGraph { cudaGraph_t graph;cudaGraphExec_t exec;int device; };-struct Dsv4CudaKvCache { float *kv,*compressed,*comp_value,*comp_score,*rope_cos,*rope_sin,*comp_cos,*comp_sin,*fi_lse,*fi_out_lse,*fi_cos_sin;uint8_t *fi_kv,*fi_comp;__nv_bfloat16 *fi_q,*fi_mid,*fi_out;int *compressed_len,*fi_indices,*fi_comp_indices,*fi_length,*fi_comp_length;int64_t *fi_slot,*fi_pos;size_t fi_qcap,fi_midcap,fi_outcap,fi_lsecap,fi_out_lsecap,fi_indicescap,fi_lengthcap,fi_slotcap,fi_poscap;int device,window,dim,max_tokens,max_compressed,rope_pairs,fi_splits; };+struct Dsv4CudaKvCache { float *kv,*compressed,*comp_value,*comp_score,*rope_cos,*rope_sin,*comp_cos,*comp_sin;int *compressed_len;int device,window,dim,max_tokens,max_compressed,rope_pairs; };struct MvDesc { const uint8_t *w,*scale; const float *x; float *y; };-struct ExpertPtr { const uint8_t *w,*scale,*fi_scale; };+struct ExpertPtr { const uint8_t *w,*scale; };struct Dsv4CudaExpertSet { ExpertPtr *table;int64_t *hash;Dsv4CudaTensor *sg,*su,*sd;uint8_t *bank_fc1,*bank_fc1_scale,*bank_fc2,*bank_fc2_scale;int *bank_fc1_dg_scale,*bank_fc2_dg_scale;int count,device,H,I,vocab,topk; };struct TcPlan {int O,I,ok;cublasLtMatmulDesc_t op;cublasLtMatrixLayout_t a,b,c,d;cublasLtMatmulHeuristicResult_t algo;};struct Dev { int id; cudaStream_t stream,aux; cudaEvent_t ready,fork,join;float *dx,*dy,*p1,*p2,*p3,*p4,*aux1,*aux2,*aux3; size_t xcap,ycap,p1cap,p2cap,p3cap,p4cap,aux1cap,aux2cap,aux3cap; float *hx,*hy;#ifdef COLI_DSV4_NCCLncclComm_t ep_comm;-#endif-#ifdef COLI_DSV4_VLLM_MHC- __nv_bfloat16 *mhcb1,*mhcb2,*mhcb3;size_t mhcb1cap,mhcb2cap,mhcb3cap;#endifint *decode_state;MvDesc *mvdesc;size_t mvdesccap;float *expert_weights;size_t expert_weightscap;int *expert_ids;size_t expert_idcap;@@ -56,17 +46,8 @@ static int nok(ncclResult_t e,const char*s){if(e==ncclSuccess)return 1;fprintf(s#endifstatic int buf(void **p,size_t *cap,size_t n){if(*cap>=n)return 1;if(*p)cudaFree(*p);*p=nullptr;*cap=0;if(!ok(cudaMalloc(p,n),"buffer allocation"))return 0;*cap=n;return 1;}-#ifdef COLI_DSV4_VLLM_MHC-__global__ void mhc_float_to_bf16(__nv_bfloat16*out,const float*in,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=__float2bfloat16(in[i]);}-__global__ void mhc_bf16_to_float(float*out,const __nv_bfloat16*in,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=__bfloat162float(in[i]);}-#endif-__constant__ float e8_table[256];__global__ void set_decode_state(int*state,int token,int position){if(!threadIdx.x){state[0]=token;state[1]=position;}}-#ifdef COLI_DSV4_FLASHINFER-__device__ __forceinline__ int fi_sf_off(int row,int col,int cols){return (col&3)+(col/4)*512+(row&31)*16+((row&127)/32)*4+(row/128)*128*((cols+3)&~3);}-__global__ void fi_swizzle_scale(uint8_t*out,const uint8_t*in,int O,int cols){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)O*cols;if(p<n){int row=p/cols,col=p%cols;out[fi_sf_off(row,col,cols)]=in[p];}}-#endif__device__ float e4m3(uint8_t b){int sign=b>>7,e=(b>>3)&15,m=b&7;float v;if(!e)v=ldexpf((float)m,-9);else if(e==15)v=m==7?NAN:ldexpf(1.f+m/8.f,8);else v=ldexpf(1.f+m/8.f,e-7);return sign?-v:v;}@@ -579,24 +560,8 @@ __device__ __forceinline__ int decode_pos(const int *state){return state[1];}__device__ __forceinline__ int compression_ready(const int *state,int ratio){return ratio&&decode_pos(state)%ratio==ratio-1;}__global__ void rope_heads_dynamic(float*x,int heads,int dim,int rope,const int*state,int delta,int inverse,int predicate,const float*cs,const float*sn){if(predicate&&!compression_ready(state,predicate))return;int pos=decode_pos(state)+delta,p=blockIdx.x*blockDim.x+threadIdx.x,pairs=rope/2;if(p<heads*pairs){int h=p/pairs,k=p%pairs,off=h*dim+dim-rope+2*k;float s=inverse?-sn[(long long)pos*pairs+k]:sn[(long long)pos*pairs+k],u=x[off],v=x[off+1];x[off]=__bfloat162float(__float2bfloat16(u*cs[(long long)pos*pairs+k]-v*s));x[off+1]=__bfloat162float(__float2bfloat16(u*s+v*cs[(long long)pos*pairs+k]));}}__global__ void cache_store_dynamic(float*cache,const float*kv,const int*state,int window,int dim){int d=blockIdx.x*blockDim.x+threadIdx.x;if(d<dim)cache[(long long)(decode_pos(state)%window)*dim+d]=kv[d];}-#ifdef COLI_DSV4_FLASHINFER-__global__ void fi_prepare_q(__nv_bfloat16*q,const float*x,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)q[i]=__float2bfloat16(x[i]);}-__global__ void rmsnorm_rows(float*out,const float*in,const float*w,int rows,int H,float eps){int row=blockIdx.x;const float*x=in+(long long)row*H;float*y=out+(long long)row*H;__shared__ float ws[8],inv;int lane=threadIdx.x&31,warp=threadIdx.x>>5;float ss=0;for(int h=threadIdx.x;h<H;h+=blockDim.x){float v=x[h];ss+=v*v;}for(int d=16;d;d>>=1)ss+=__shfl_down_sync(0xffffffff,ss,d);if(!lane)ws[warp]=ss;__syncthreads();if(!warp){float v=lane<8?ws[lane]:0;for(int d=16;d;d>>=1)v+=__shfl_down_sync(0xffffffff,v,d);if(!lane)inv=rsqrtf(v/H+eps);}__syncthreads();for(int h=threadIdx.x;h<H;h+=blockDim.x)y[h]=__bfloat162float(__float2bfloat16(x[h]*inv*w[h]));(void)rows;}-__global__ void rmsnorm_rows_bf16(float*out,const float*in,const __nv_bfloat16*w,int rows,int H,float eps){int row=blockIdx.x;const float*x=in+(long long)row*H;float*y=out+(long long)row*H;__shared__ float ws[8],inv;int lane=threadIdx.x&31,warp=threadIdx.x>>5;float ss=0;for(int h=threadIdx.x;h<H;h+=blockDim.x){float v=x[h];ss+=v*v;}for(int d=16;d;d>>=1)ss+=__shfl_down_sync(0xffffffff,ss,d);if(!lane)ws[warp]=ss;__syncthreads();if(!warp){float v=lane<8?ws[lane]:0;for(int d=16;d;d>>=1)v+=__shfl_down_sync(0xffffffff,v,d);if(!lane)inv=rsqrtf(v/H+eps);}__syncthreads();for(int h=threadIdx.x;h<H;h+=blockDim.x)y[h]=__bfloat162float(__float2bfloat16(x[h]*inv*__bfloat162float(w[h])));(void)rows;}-__global__ void split_qkv_rows(float*q,float*kv,const float*in,int rows,int R,int K){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)rows*(R+K);if(p<n){int row=p/(R+K),col=p%(R+K);if(col<R)q[(long long)row*R+col]=in[p];else kv[(long long)row*K+col-R]=in[p];}}-__global__ void fi_position_batch(int64_t*slot,int64_t*position,int start,int tokens){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<tokens)slot[i]=position[i]=start+i;}-__global__ void fi_indices_batch(int*idx,int*length,int start,int tokens,int window,int topk){int t=blockIdx.x,p=threadIdx.x;if(t>=tokens)return;int pos=start+t,n=min(pos+1,window),first=pos+1-n;if(p<topk)idx[(long long)t*topk+p]=p<n?first+p:-1;if(!p)length[t]=n;}__global__ void gather_group_rows(float*out,const float*in,int tokens,int groups,int width,int group){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)tokens*width;if(p<n){int row=p/width,col=p%width;out[p]=in[((long long)row*groups+group)*width+col];}}__global__ void scatter_group_rows(float*out,const float*in,int tokens,int groups,int width,int group){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)tokens*width;if(p<n){int row=p/width,col=p%width;out[((long long)row*groups+group)*width+col]=in[p];}}-__global__ void fi_indices(int*idx,int*length,const int*state,int window){int i=threadIdx.x,pos=decode_pos(state),n=min(pos+1,window),start=pos+1-n;if(i<512)idx[i]=i<n?start+i:-1;if(!i)*length=n;}-/* C4A attends to the indexer's compressed candidates in addition to SWA.- * The ranges intentionally overlap: those summaries are separate learned KV- * entries, not duplicates for the runtime to remove. Before the Lightning- * Indexer is needed, every live compressed entry is therefore a candidate. */-__global__ void fi_extra_indices(int*idx,int*out_length,const int*length,const int*state,int window,int ratio){int i=threadIdx.x,n=min(*length,512);if(i<512)idx[i]=i<n?i:-1;if(!i)*out_length=n;(void)state;(void)window;(void)ratio;}-__global__ void fi_position(int64_t*slot,int64_t*position,const int*state){if(!threadIdx.x){*slot=decode_pos(state);*position=decode_pos(state);}}-__global__ void fi_cos_sin(float*out,const float*cs,const float*sn,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){int pos=i/32,k=i%32;out[(long long)pos*64+k]=cs[i];out[(long long)pos*64+32+k]=sn[i];}}-#endif__global__ void compress_store_dynamic(float*values,float*scores,const float*v,const float*g,const float*ape,const int*state,int ratio,int overlap,int O){int d=blockIdx.x*blockDim.x+threadIdx.x;if(d<O){int phase=decode_pos(state)%ratio,slot=(overlap?ratio:0)+phase;values[(long long)slot*O+d]=v[d];scores[(long long)slot*O+d]=g[d]+ape[(long long)phase*O+d];}}__global__ void compress_pool_dynamic(float*out,const float*values,const float*scores,const int*state,int ratio,int overlap,int dim){if(!compression_ready(state,ratio))return;int d=blockIdx.x*blockDim.x+threadIdx.x;if(d<dim){int O=(1+overlap)*dim,items=overlap?2*ratio:ratio;float mx=-INFINITY;for(int j=0;j<items;j++){int col=overlap&&j>=ratio?dim+d:d;mx=fmaxf(mx,scores[(long long)j*O+col]);}float den=0,sum=0;for(int j=0;j<items;j++){int col=overlap&&j>=ratio?dim+d:d,aoff=j*O+col;float a=expf(scores[aoff]-mx);den+=a;sum+=a*values[aoff];}out[d]=sum/den;}}__global__ void rmsnorm_dynamic(float*out,const float*x,const float*w,int n,float eps,const int*state,int ratio){if(!compression_ready(state,ratio))return;__shared__ float sum[256],inv;float ss=0;for(int i=threadIdx.x;i<n;i+=blockDim.x)ss+=x[i]*x[i];sum[threadIdx.x]=ss;__syncthreads();for(int k=128;k;k>>=1){if(threadIdx.x<k)sum[threadIdx.x]+=sum[threadIdx.x+k];__syncthreads();}if(!threadIdx.x)inv=rsqrtf(sum[0]/n+eps);__syncthreads();for(int i=threadIdx.x;i<n;i+=blockDim.x)out[i]=__bfloat162float(__float2bfloat16(x[i]*inv*w[i]));}@@ -638,13 +603,8 @@ __global__ void build_moe_desc(MvDesc *desc,const ExpertPtr *table,const int *idfloat *gate,float *up,float *act,float *down,int count,int experts,int H,int I){int k=threadIdx.x;if(k>=count)return;int e=ids[k];if(e<0||e>=experts)return;ExpertPtr g=table[e],u=table[experts+e],d=table[2*experts+e];-#ifdef COLI_DSV4_FLASHINFER- desc[2*k]={g.w,g.fi_scale,x,gate+(long long)k*I};desc[2*k+1]={u.w,u.fi_scale,x,up+(long long)k*I};- desc[2*count+k]={d.w,d.fi_scale,act+(long long)k*I,down+(long long)k*H};-#elsedesc[2*k]={g.w,g.scale,x,gate+(long long)k*I};desc[2*k+1]={u.w,u.scale,x,up+(long long)k*I};desc[2*count+k]={d.w,d.scale,act+(long long)k*I,down+(long long)k*H};-#endif}__global__ void load_hash_ids(int *ids,const int64_t *map,const int *state,int topk){int k=threadIdx.x;if(k<topk)ids[k]=(int)map[(long long)state[0]*topk+k];}__global__ void mhc_mix_scale(float *mix,const float *residual,const float *scale,const float *base,int M,int H,float eps){@@ -688,18 +648,12 @@ extern "C" int dsv4_cuda_init(const int *dev,int n){if(n<1||n>16)return 0;floatreturn 1;}static void plan_free(TcPlan*p){if(p->d)cublasLtMatrixLayoutDestroy(p->d);if(p->c)cublasLtMatrixLayoutDestroy(p->c);if(p->b)cublasLtMatrixLayoutDestroy(p->b);if(p->a)cublasLtMatrixLayoutDestroy(p->a);if(p->op)cublasLtMatmulDescDestroy(p->op);memset(p,0,sizeof(*p));}extern "C" void dsv4_cuda_shutdown(void){-#ifdef COLI_DSV4_FLASHINFER- dsv4_flashinfer_shutdown();-#endiffor(int i=0;i<ng;i++){cudaSetDevice(g[i].id);if(g[i].decode_state)cudaFree(g[i].decode_state);#ifdef COLI_DSV4_NCCLif(g[i].ep_comm)ncclCommDestroy(g[i].ep_comm);#endif}for(int i=0;i<ng;i++){cudaSetDevice(g[i].id);if(g[i].dx)cudaFree(g[i].dx);if(g[i].dy)cudaFree(g[i].dy);if(g[i].p1)cudaFree(g[i].p1);if(g[i].p2)cudaFree(g[i].p2);if(g[i].p3)cudaFree(g[i].p3);if(g[i].p4)cudaFree(g[i].p4);if(g[i].aux1)cudaFree(g[i].aux1);if(g[i].aux2)cudaFree(g[i].aux2);if(g[i].aux3)cudaFree(g[i].aux3);if(g[i].mvdesc)cudaFree(g[i].mvdesc);if(g[i].expert_weights)cudaFree(g[i].expert_weights);if(g[i].expert_ids)cudaFree(g[i].expert_ids);if(g[i].tcw)cudaFree(g[i].tcw);if(g[i].tcs)cudaFree(g[i].tcs);if(g[i].tcx)cudaFree(g[i].tcx);if(g[i].tcxs)cudaFree(g[i].tcxs);if(g[i].tc_workspace)cudaFree(g[i].tc_workspace);-#ifdef COLI_DSV4_VLLM_MHC- if(g[i].mhcb1)cudaFree(g[i].mhcb1);if(g[i].mhcb2)cudaFree(g[i].mhcb2);if(g[i].mhcb3)cudaFree(g[i].mhcb3);-#endif#ifdef COLI_DSV4_DEEPGEMMif(g[i].dga1)cudaFree(g[i].dga1);if(g[i].dga2)cudaFree(g[i].dga2);if(g[i].dgsfa1)cudaFree(g[i].dgsfa1);if(g[i].dgsfa2)cudaFree(g[i].dgsfa2);if(g[i].dglayout)cudaFree(g[i].dglayout);if(g[i].dgmm1)cudaFree(g[i].dgmm1);if(g[i].dgmm2)cudaFree(g[i].dgmm2);if(g[i].dgref)cudaFree(g[i].dgref);if(g[i].dgdensews)cudaFree(g[i].dgdensews);#endif@@ -748,9 +702,6 @@ static int upload(Dsv4CudaTensor **pt,const void *w,size_t wb,const uint8_t *sc,extern "C" int dsv4_cuda_upload_fp8(Dsv4CudaTensor **t,const uint8_t*w,const uint8_t*s,int O,int I,int d){return upload(t,w,(size_t)O*I,s,(size_t)((O+127)/128)*((I+127)/128),O,I,d,8);}extern "C" int dsv4_cuda_upload_fp8_bf16(Dsv4CudaTensor **t,const uint8_t*w,const uint8_t*s,int O,int I,int d){return upload(t,w,(size_t)O*I,s,(size_t)((O+127)/128)*((I+127)/128),O,I,d,9);}extern "C" int dsv4_cuda_upload_fp4(Dsv4CudaTensor **t,const uint8_t*w,const uint8_t*s,int O,int I,int d){if(!upload(t,w,(size_t)O*I/2,s,(size_t)O*I/32,O,I,d,4))return 0;-#ifdef COLI_DSV4_FLASHINFER- size_t bytes=(size_t)((O+127)&~127)*((I/32+3)&~3);if(!ok(cudaMalloc(&(*t)->fi_scale,bytes),"FlashInfer scale allocation")||!ok(cudaMemset((*t)->fi_scale,0,bytes),"FlashInfer scale clear"))return 0;fi_swizzle_scale<<<((long long)O*(I/32)+255)/256,256>>>((*t)->fi_scale,(*t)->scale,O,I/32);if(!ok(cudaGetLastError(),"FlashInfer scale swizzle"))return 0;(*t)->bytes+=bytes;-#endifreturn 1;}extern "C" int dsv4_cuda_upload_bf16(Dsv4CudaTensor **t,const uint16_t*w,int O,int I,int d){return upload(t,w,(size_t)O*I*2,nullptr,0,O,I,d,16);}extern "C" int dsv4_cuda_upload_f32(Dsv4CudaTensor **t,const float*w,int O,int I,int d){return upload(t,w,(size_t)O*I*4,nullptr,0,O,I,d,32);}@@ -828,18 +779,6 @@ extern "C" Dsv4CudaKvCache *dsv4_cuda_kv_create(int device, int window, int dim,dsv4_cuda_kv_free(k);return nullptr;}-#ifdef COLI_DSV4_FLASHINFER- size_t fib=(size_t)((max_tokens+63)/64)*64*584,fic=(size_t)((k->max_compressed+63)/64)*64*584;- k->fi_splits=(window+63)/64+(k->max_compressed+63)/64;- if(!ok(cudaMalloc(&k->fi_kv,fib),"FlashInfer KV allocation")||!ok(cudaMemset(k->fi_kv,0,fib),"FlashInfer KV clear")||- !ok(cudaMalloc(&k->fi_comp,fic),"FlashInfer compressed KV allocation")||!ok(cudaMemset(k->fi_comp,0,fic),"FlashInfer compressed KV clear")||- !ok(cudaMalloc(&k->fi_indices,512*sizeof(int)),"FlashInfer indices allocation")||!ok(cudaMalloc(&k->fi_comp_indices,512*sizeof(int)),"FlashInfer compressed indices allocation")||!ok(cudaMalloc(&k->fi_length,sizeof(int)),"FlashInfer length allocation")||!ok(cudaMalloc(&k->fi_comp_length,sizeof(int)),"FlashInfer compressed length allocation")||- !ok(cudaMalloc(&k->fi_q,64*512*sizeof(__nv_bfloat16)),"FlashInfer Q allocation")||!ok(cudaMalloc(&k->fi_mid,(size_t)64*k->fi_splits*512*sizeof(__nv_bfloat16)),"FlashInfer split output allocation")||- !ok(cudaMalloc(&k->fi_lse,(size_t)64*k->fi_splits*sizeof(float)),"FlashInfer split LSE allocation")||!ok(cudaMalloc(&k->fi_out,64*512*sizeof(__nv_bfloat16)),"FlashInfer output allocation")||- !ok(cudaMalloc(&k->fi_out_lse,64*sizeof(float)),"FlashInfer output LSE allocation")||!ok(cudaMalloc(&k->fi_slot,sizeof(int64_t)),"vLLM slot allocation")||- !ok(cudaMalloc(&k->fi_pos,sizeof(int64_t)),"vLLM position allocation")||!ok(cudaMalloc(&k->fi_cos_sin,(size_t)max_tokens*64*sizeof(float)),"vLLM RoPE cache allocation")){dsv4_cuda_kv_free(k);return nullptr;}- fi_cos_sin<<<(max_tokens*rope_pairs+255)/256,256,0,c->stream>>>(k->fi_cos_sin,k->rope_cos,k->rope_sin,max_tokens*rope_pairs);-#endiffill_value<<<(pb / 4 + 255) / 256, 256, 0, c->stream>>>(k->comp_score, pb / 4, -INFINITY);if (!ok(cudaGetLastError(), "compress score clear")) {dsv4_cuda_kv_free(k);@@ -878,43 +817,10 @@ extern "C" int dsv4_cuda_mhc_post_pre(const Dsv4CudaActivation*x,const Dsv4CudaA}extern "C" int dsv4_cuda_mhc_post_pre_norm(const Dsv4CudaActivation*x,const Dsv4CudaActivation*r,Dsv4CudaActivation*state,int M,int H,Dsv4CudaActivation*out,Dsv4CudaTensor*fn,Dsv4CudaTensor*scale,Dsv4CudaTensor*base,Dsv4CudaTensor*norm,float rms_eps,float pre_eps,float sink_eps,float post_mult,int sink_iters,float norm_eps,Dsv4CudaActivation*input){constexpr int N=24,parts=8;int I=M*H;Dev*c=x?ctx(x->device):nullptr;if(!c||!r||!state||!out||!fn||!scale||!base||!norm||!input||M!=4||r->device!=x->device||state->device!=x->device||out->device!=x->device||fn->device!=x->device||scale->device!=x->device||base->device!=x->device||norm->device!=x->device||input->device!=x->device||x->elements<H||r->elements<I||state->elements<M+M*M+M||out->elements<I||fn->fmt!=32||norm->fmt!=32||fn->O!=N||fn->I!=I||norm->O*norm->I<H||!ok(cudaSetDevice(x->device),"select fused normalized mHC device")||!buf((void**)&c->p1,&c->p1cap,N*sizeof(float))||!buf((void**)&c->dy,&c->ycap,(N+1)*parts*sizeof(float)))return 0;-#ifdef COLI_DSV4_VLLM_MHC- int vllm=getenv("DSV4_CUDA_VLLM_MHC")&&atoi(getenv("DSV4_CUDA_VLLM_MHC"));- if(vllm&&H==4096&&rms_eps==1e-6f&&pre_eps==1e-6f&&sink_eps==1e-6f&&post_mult==2.f&&sink_iters==20&&norm_eps==1e-6f){- if(!buf((void**)&c->mhcb1,&c->mhcb1cap,(size_t)I*sizeof(__nv_bfloat16))||!buf((void**)&c->mhcb2,&c->mhcb2cap,(size_t)H*sizeof(__nv_bfloat16))||!buf((void**)&c->mhcb3,&c->mhcb3cap,(size_t)I*sizeof(__nv_bfloat16)))return 0;- if(!norm->bf16_cache){if(!ok(cudaMalloc(&norm->bf16_cache,(size_t)H*sizeof(__nv_bfloat16)),"mHC norm BF16 cache allocation"))return 0;mhc_float_to_bf16<<<(H+255)/256,256,0,c->stream>>>((__nv_bfloat16*)norm->bf16_cache,(float*)norm->w,H);}- mhc_float_to_bf16<<<(I+255)/256,256,0,c->stream>>>(c->mhcb1,r->data,I);mhc_float_to_bf16<<<(H+255)/256,256,0,c->stream>>>(c->mhcb2,x->data,H);- cudaError_t e=dsv4_vllm_mhc_post_pre_norm(c->stream,state->data+M,c->mhcb1,state->data,c->mhcb2,(float*)fn->w,c->dy,c->dy+N*parts,c->mhcb3,(float*)scale->w,(float*)base->w,c->mhcb2,norm->bf16_cache);- if(e!=cudaSuccess)return ok(e,"vLLM mHC launch");mhc_bf16_to_float<<<(I+255)/256,256,0,c->stream>>>(out->data,c->mhcb3,I);mhc_bf16_to_float<<<(H+255)/256,256,0,c->stream>>>(input->data,c->mhcb2,H);return ok(cudaGetLastError(),"vLLM mHC conversion launch");- }-#endifmhc_post_fn_split<<<parts,256,0,c->stream>>>((float*)fn->w,x->data,r->data,state->data,out->data,c->dy,N,M,H,parts);mhc_fn_finish<<<1,32,0,c->stream>>>(c->p1,c->dy,(float*)scale->w,(float*)base->w,N,M,I,rms_eps,parts);mhc_coeff<<<1,32,0,c->stream>>>(state->data,c->p1,M,pre_eps,sink_eps,post_mult,sink_iters);mhc_input_norm<<<1,256,0,c->stream>>>(input->data,out->data,state->data,(float*)norm->w,M,H,norm_eps);return ok(cudaGetLastError(),"fused normalized mHC post/pre launch");}extern "C" int dsv4_cuda_mhc_post_pre_norm_batch(const Dsv4CudaActivation*x,const Dsv4CudaActivation*r,Dsv4CudaActivation*state,int tokens,int H,Dsv4CudaActivation*out,Dsv4CudaTensor*fn,Dsv4CudaTensor*scale,Dsv4CudaTensor*base,Dsv4CudaTensor*norm,Dsv4CudaActivation*input){-#ifdef COLI_DSV4_VLLM_MHC- constexpr int M=4,N=24,parts=8;long long RH=(long long)tokens*M*H,XH=(long long)tokens*H;- Dev*c=x?ctx(x->device):nullptr;if(!c||!r||!state||!out||!fn||!scale||!base||!norm||!input||tokens<1||H!=4096||RH>INT_MAX||XH>INT_MAX||- r->device!=x->device||state->device!=x->device||out->device!=x->device||fn->device!=x->device||- scale->device!=x->device||base->device!=x->device||norm->device!=x->device||input->device!=x->device||- x->elements<XH||r->elements<RH||state->elements<(long long)tokens*(M+M*M)||out->elements<RH||input->elements<XH||- fn->fmt!=32||norm->fmt!=32||fn->O!=N||fn->I!=M*H||norm->O*norm->I<H||- !ok(cudaSetDevice(x->device),"select batched mHC device")||- !buf((void**)&c->mhcb1,&c->mhcb1cap,(size_t)RH*sizeof(__nv_bfloat16))||- !buf((void**)&c->mhcb2,&c->mhcb2cap,(size_t)XH*sizeof(__nv_bfloat16))||- !buf((void**)&c->mhcb3,&c->mhcb3cap,(size_t)RH*sizeof(__nv_bfloat16))||- !buf((void**)&c->dy,&c->ycap,(size_t)tokens*parts*(N+1)*sizeof(float)))return 0;- if(!norm->bf16_cache){if(!ok(cudaMalloc(&norm->bf16_cache,(size_t)H*sizeof(__nv_bfloat16)),"mHC norm BF16 cache allocation"))return 0;mhc_float_to_bf16<<<(H+255)/256,256,0,c->stream>>>((__nv_bfloat16*)norm->bf16_cache,(float*)norm->w,H);}- mhc_float_to_bf16<<<(RH+255)/256,256,0,c->stream>>>(c->mhcb1,r->data,(int)RH);- mhc_float_to_bf16<<<(XH+255)/256,256,0,c->stream>>>(c->mhcb2,x->data,(int)XH);- float*post=state->data,*comb=post+(long long)tokens*M,*mul=c->dy,*sq=mul+(long long)tokens*parts*N;- cudaError_t e=dsv4_vllm_mhc_post_pre_norm_batch(c->stream,comb,c->mhcb1,post,c->mhcb2,(float*)fn->w,mul,sq,c->mhcb3,(float*)scale->w,(float*)base->w,c->mhcb2,norm->bf16_cache,tokens);- if(e!=cudaSuccess)return ok(e,"vLLM batched mHC launch");- mhc_bf16_to_float<<<(RH+255)/256,256,0,c->stream>>>(out->data,c->mhcb3,(int)RH);- mhc_bf16_to_float<<<(XH+255)/256,256,0,c->stream>>>(input->data,c->mhcb2,(int)XH);- return ok(cudaGetLastError(),"vLLM batched mHC conversion launch");-#else(void)x;(void)r;(void)state;(void)tokens;(void)H;(void)out;(void)fn;(void)scale;(void)base;(void)norm;(void)input;return 0;-#endif}extern "C" int dsv4_cuda_mhc_post_batch(const Dsv4CudaActivation*x,const Dsv4CudaActivation*r,const Dsv4CudaActivation*state,int tokens,int H,Dsv4CudaActivation*out){long long RH=(long long)tokens*4*H,XH=(long long)tokens*H;Dev*c=x?ctx(x->device):nullptr;if(!c||!r||!state||!out||tokens<1||H!=4096||r->device!=x->device||state->device!=x->device||out->device!=x->device||x->elements<XH||r->elements<RH||state->elements<(long long)tokens*20||out->elements<RH||!ok(cudaSetDevice(x->device),"select batched mHC output device"))return 0;mhc_output_batch<<<(RH+255)/256,256,0,c->stream>>>(out->data,x->data,r->data,state->data,state->data+(long long)tokens*4,H);return ok(cudaGetLastError(),"batched mHC output launch");@@ -931,9 +837,6 @@ extern "C" void dsv4_cuda_kv_free(Dsv4CudaKvCache *k) {if (k->rope_sin) cudaFree(k->rope_sin);if (k->comp_cos) cudaFree(k->comp_cos);if (k->comp_sin) cudaFree(k->comp_sin);-#ifdef COLI_DSV4_FLASHINFER- cudaFree(k->fi_kv);cudaFree(k->fi_comp);cudaFree(k->fi_indices);cudaFree(k->fi_comp_indices);cudaFree(k->fi_length);cudaFree(k->fi_comp_length);cudaFree(k->fi_q);cudaFree(k->fi_mid);cudaFree(k->fi_lse);cudaFree(k->fi_out);cudaFree(k->fi_out_lse);cudaFree(k->fi_slot);cudaFree(k->fi_pos);cudaFree(k->fi_cos_sin);-#endiffree(k);}extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4CudaTensor *an, Dsv4CudaTensor *qa,@@ -991,15 +894,9 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4Cif (qk_rope)rope_heads_dynamic<<<(qk_rope / 2 + 255) / 256, 256, 0, comp>>>(comp3, 1, dim, qk_rope, c->decode_state, 1 - ratio, 0, ratio, cache->comp_cos, cache->comp_sin);- /* The FlashInfer cache packer performs the model's UE8M0 FP8- * quantization. Quantizing and dequantizing here first would apply- * a second, differently-scaled FP8 round before the real cache- * store. Keep the simulation only for the float-cache fallback. */-#ifndef COLI_DSV4_FLASHINFERif (dim > qk_rope)fp8_sim64_dynamic<<<(dim - qk_rope + 63) / 64, 64, 0, comp>>>(comp3, dim - qk_rope,c->decode_state, ratio);-#endifcompressed_store_dynamic<<<(dim + 255) / 256, 256, 0, comp>>>(cache->compressed, comp3, cache->compressed_len, c->decode_state, ratio, cache->max_compressed, dim);int commit_blocks = overlap ? (ratio * O + 255) / 256 : 1;@@ -1022,12 +919,8 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4C{fp8_sim<<<(R + 127) / 128, 128, 0, c->stream>>>(c->p1, 1, R);run_mv<8>((uint8_t *)qb->w, qb->scale, c->p1, c->p2, Q, R, 1, c->stream);}bf16_round<<<(Q + 255) / 256, 256, 0, c->stream>>>(c->p2, Q);- int sparse=0;-#ifdef COLI_DSV4_FLASHINFER- sparse=getenv("DSV4_CUDA_SPARSE_MLA")&&atoi(getenv("DSV4_CUDA_SPARSE_MLA"))&&(heads==8||heads==16||heads==32||heads==64)&&dim==512&&cache->max_compressed<=512;-#endif- if(!sparse)head_rmsnorm<<<heads, 256, 0, c->stream>>>(c->p2, heads, dim, eps);- if (qk_rope&&!sparse)+ head_rmsnorm<<<heads, 256, 0, c->stream>>>(c->p2, heads, dim, eps);+ if (qk_rope)rope_heads_dynamic<<<(heads * (qk_rope / 2) + 255) / 256, 256, 0, c->stream>>>(c->p2, heads, dim, qk_rope, c->decode_state, 0, 0, 0, cache->rope_cos, cache->rope_sin);#ifdef COLI_DSV4_DEEPGEMM@@ -1036,25 +929,12 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4Crun_mv<8>((uint8_t *)kv->w, kv->scale, c->dx, c->p3, K, H, 1, c->stream);if (!fused_qkv) bf16_round<<<(K + 255) / 256, 256, 0, c->stream>>>(c->p3, K);rmsnorm_f32<<<1, 256, 0, c->stream>>>(c->p3, fused_qkv ? c->p1 + R : c->p3, (float *)kvn->w, K, eps);- if (qk_rope&&!sparse)+ if (qk_rope)rope_heads_dynamic<<<(qk_rope / 2 + 255) / 256, 256, 0, c->stream>>>(c->p3, 1, dim, qk_rope, c->decode_state, 0, 0, 0, cache->rope_cos, cache->rope_sin);- if(!sparse){if (K > qk_rope) fp8_sim64<<<(K - qk_rope + 63) / 64, 64, 0, c->stream>>>(c->p3, K - qk_rope);cache_store_dynamic<<<(K + 255) / 256, 256, 0, c->stream>>>(cache->kv, c->p3, c->decode_state, cache->window, K);}-#ifdef COLI_DSV4_FLASHINFER- else{fi_prepare_q<<<(heads*dim+255)/256,256,0,c->stream>>>(cache->fi_mid,c->p2,heads*dim);fi_prepare_q<<<(dim+255)/256,256,0,c->stream>>>(cache->fi_out,c->p3,dim);fi_position<<<1,1,0,c->stream>>>(cache->fi_slot,cache->fi_pos,c->decode_state);- if(!dsv4_vllm_qnorm_rope_kv_insert(cache->fi_mid,cache->fi_q,cache->fi_out,cache->fi_kv,cache->fi_slot,cache->fi_pos,cache->fi_cos_sin,eps,heads,64,64*584,c->stream))return 0;}-#endif+ if (K > qk_rope) fp8_sim64<<<(K - qk_rope + 63) / 64, 64, 0, c->stream>>>(c->p3, K - qk_rope);+ cache_store_dynamic<<<(K + 255) / 256, 256, 0, c->stream>>>(cache->kv, c->p3, c->decode_state, cache->window, K);if (async && !ok(cudaStreamWaitEvent(c->stream, c->join, 0), "attention join wait")) return 0;-#ifdef COLI_DSV4_FLASHINFER- if(sparse){int main_topk=((cache->window+63)/64)*64,main_splits=main_topk/64;- int use_extra=ratio&&(pos+1)>=ratio;- int extra_cap=use_extra?(cache->max_tokens+ratio-1)/ratio:0,extra_topk=((extra_cap+63)/64)*64,extra_splits=extra_topk/64;- if(use_extra&&!dsv4_vllm_pack_compressed(cache->compressed,cache->fi_comp,c->decode_state,ratio,64,64*584,c->stream))return 0;- fi_indices<<<1,512,0,c->stream>>>(cache->fi_indices,cache->fi_length,c->decode_state,cache->window);if(use_extra)fi_extra_indices<<<1,512,0,c->stream>>>(cache->fi_comp_indices,cache->fi_comp_length,cache->compressed_len,c->decode_state,cache->window,ratio);- if(!dsv4_flashinfer_sparse_mla(cache->fi_q,cache->fi_kv,cache->fi_indices,cache->fi_mid,cache->fi_lse,cache->fi_out,cache->fi_out_lse,cache->fi_length,(float*)sink->w,use_extra?cache->fi_comp:nullptr,use_extra?cache->fi_comp_indices:nullptr,use_extra?cache->fi_comp_length:nullptr,extra_topk,use_extra?64:0,use_extra?64*584:0,heads,main_topk,1,main_splits+extra_splits,0,1.f/sqrtf((float)dim),64*584,c->stream))return 0;- mhc_bf16_to_float<<<(heads*dim+255)/256,256,0,c->stream>>>(c->p4,cache->fi_out,heads*dim);- }else-#endifattention_window_dynamic<<<heads, 256, (size_t)(cache->window + cache->max_compressed) * 4, c->stream>>>(c->p4, c->p2, cache->kv, cache->compressed, (float *)sink->w, heads, dim, cache->window, c->decode_state,cache->compressed_len);@@ -1075,55 +955,8 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4Creturn ok(cudaGetLastError(), "attention window launch");}-static int fi_reserve_batch(Dsv4CudaKvCache*k,int tokens,int heads,int topk,int splits){-#ifdef COLI_DSV4_FLASHINFER- size_t q=(size_t)tokens*heads*512*sizeof(__nv_bfloat16),mid=(size_t)tokens*heads*splits*512*sizeof(__nv_bfloat16),- lse=(size_t)tokens*heads*splits*sizeof(float),out_lse=(size_t)tokens*heads*sizeof(float),- indices=(size_t)tokens*topk*sizeof(int),length=(size_t)tokens*sizeof(int),pos=(size_t)tokens*sizeof(int64_t);- return buf((void**)&k->fi_q,&k->fi_qcap,q)&&buf((void**)&k->fi_mid,&k->fi_midcap,mid)&&- buf((void**)&k->fi_out,&k->fi_outcap,q)&&buf((void**)&k->fi_lse,&k->fi_lsecap,lse)&&- buf((void**)&k->fi_out_lse,&k->fi_out_lsecap,out_lse)&&buf((void**)&k->fi_indices,&k->fi_indicescap,indices)&&- buf((void**)&k->fi_length,&k->fi_lengthcap,length)&&buf((void**)&k->fi_slot,&k->fi_slotcap,pos)&&- buf((void**)&k->fi_pos,&k->fi_poscap,pos);-#else- (void)k;(void)tokens;(void)heads;(void)topk;(void)splits;return 0;-#endif-}-extern "C" int dsv4_cuda_attention_sparse_batch(const Dsv4CudaActivation*input,Dsv4CudaTensor*an,Dsv4CudaTensor*qkv,Dsv4CudaTensor*qn,Dsv4CudaTensor*qb,Dsv4CudaTensor*kvn,Dsv4CudaTensor*sink,int heads,int dim,int start,int tokens,float eps,Dsv4CudaKvCache*cache,Dsv4CudaActivation*context){-#if defined(COLI_DSV4_FLASHINFER) && defined(COLI_DSV4_DEEPGEMM)-#define BATCH_ATTN_REQUIRE(expr,stage) do{if(!(expr)){fprintf(stderr,"[DSV4 batch attention] %s failed\n",stage);return 0;}}while(0)- Dev*c=input?ctx(input->device):nullptr;int H=qkv?qkv->I:0,K=dim,R=qkv?qkv->O-K:0,Q=heads*dim,topk=cache?max(128,((cache->window+63)/64)*64):0,splits=topk/64;- if(!c||!cache||!context||!an||!qkv||!qn||!qb||!kvn||!sink||tokens<1||start<0||start+tokens>cache->max_tokens||dim!=512||topk<64||topk>512||- cache->device!=input->device||context->device!=input->device||input->elements<(long long)tokens*H||context->elements<(long long)tokens*Q||- an->fmt!=32||qkv->fmt!=8||qn->fmt!=16||qb->fmt!=8||kvn->fmt!=32||sink->fmt!=32||qkv->O!=R+K||qb->I!=R||qb->O!=Q||- an->O*an->I<H||qn->O*qn->I<R||kvn->O*kvn->I<K||sink->O*sink->I<heads){fprintf(stderr,"[DSV4 batch attention] validation failed\n");return 0;}- Dsv4CudaTensor*tensors[]={an,qkv,qn,qb,kvn,sink};for(auto*t:tensors)if(t->device!=input->device){fprintf(stderr,"[DSV4 batch attention] device mismatch\n");return 0;}- size_t nh=(size_t)tokens*H,nr=(size_t)tokens*R,nk=(size_t)tokens*K,nq=(size_t)tokens*Q;- if(!ok(cudaSetDevice(input->device),"select batched sparse attention device")||!buf((void**)&c->dx,&c->xcap,nh*sizeof(float))||- !buf((void**)&c->p1,&c->p1cap,(size_t)tokens*(R+K)*sizeof(float))||!buf((void**)&c->p2,&c->p2cap,nr*sizeof(float))||- !buf((void**)&c->p3,&c->p3cap,nk*sizeof(float))||!buf((void**)&c->p4,&c->p4cap,nq*sizeof(float))||- !buf((void**)&c->mhcb1,&c->mhcb1cap,nq*sizeof(__nv_bfloat16))||!buf((void**)&c->mhcb2,&c->mhcb2cap,nk*sizeof(__nv_bfloat16))||- !fi_reserve_batch(cache,tokens,heads,topk,splits)){fprintf(stderr,"[DSV4 batch attention] workspace allocation failed\n");return 0;}- rmsnorm_rows<<<tokens,256,0,c->stream>>>(c->dx,input->data,(float*)an->w,tokens,H,eps);- BATCH_ATTN_REQUIRE(dg_dense_batch_dispatch(c,qkv,c->dx,c->p1,tokens),"QKV projection");- split_qkv_rows<<<((long long)tokens*(R+K)+255)/256,256,0,c->stream>>>(c->p2,c->p3,c->p1,tokens,R,K);- rmsnorm_rows_bf16<<<tokens,256,0,c->stream>>>(c->p2,c->p2,(__nv_bfloat16*)qn->w,tokens,R,eps);- BATCH_ATTN_REQUIRE(dg_dense_batch_dispatch(c,qb,c->p2,c->p4,tokens),"Q projection");- rmsnorm_rows<<<tokens,256,0,c->stream>>>(c->p3,c->p3,(float*)kvn->w,tokens,K,eps);- mhc_float_to_bf16<<<(nq+255)/256,256,0,c->stream>>>(c->mhcb1,c->p4,(int)nq);- mhc_float_to_bf16<<<(nk+255)/256,256,0,c->stream>>>(c->mhcb2,c->p3,(int)nk);- fi_position_batch<<<(tokens+255)/256,256,0,c->stream>>>(cache->fi_slot,cache->fi_pos,start,tokens);- BATCH_ATTN_REQUIRE(dsv4_vllm_qnorm_rope_kv_insert_batch(c->mhcb1,cache->fi_q,c->mhcb2,cache->fi_kv,cache->fi_slot,cache->fi_pos,cache->fi_cos_sin,eps,tokens,tokens,heads,heads,64,64*584,c->stream),"QNorm/RoPE/cache insert");- fi_indices_batch<<<tokens,topk,0,c->stream>>>(cache->fi_indices,cache->fi_length,start,tokens,cache->window,topk);- BATCH_ATTN_REQUIRE(dsv4_flashinfer_sparse_mla(cache->fi_q,cache->fi_kv,cache->fi_indices,cache->fi_mid,cache->fi_lse,cache->fi_out,cache->fi_out_lse,cache->fi_length,(float*)sink->w,nullptr,nullptr,nullptr,0,0,0,heads,topk,tokens,splits,0,1.f/sqrtf((float)dim),64*584,c->stream),"FlashInfer sparse MLA");- mhc_bf16_to_float<<<(nq+255)/256,256,0,c->stream>>>(context->data,cache->fi_out,(int)nq);- {int good=ok(cudaGetLastError(),"batched sparse attention launch");-#undef BATCH_ATTN_REQUIRE- return good;}-#else(void)input;(void)an;(void)qkv;(void)qn;(void)qb;(void)kvn;(void)sink;(void)heads;(void)dim;(void)start;(void)tokens;(void)eps;(void)cache;(void)context;return 0;-#endif}extern "C" int dsv4_cuda_attention_output_batch(const Dsv4CudaActivation*context,Dsv4CudaTensor*wa,Dsv4CudaTensor*wb,int groups,int tokens,Dsv4CudaActivation*output){@@ -1405,7 +1238,7 @@ extern "C" Dsv4CudaExpertSet *dsv4_cuda_expert_set_create(Dsv4CudaTensor *constfor(int e=0;e<count;e++){Dsv4CudaTensor*t[]={gate[e],up[e],down[e]};if(!t[0]||!t[1]||!t[2]||t[0]->device!=device||t[1]->device!=device||t[2]->device!=device||t[0]->fmt!=4||t[1]->fmt!=4||t[2]->fmt!=4||t[0]->I!=H||t[0]->O!=I||t[1]->I!=H||t[1]->O!=I||t[2]->I!=I||t[2]->O!=H){free(host);return nullptr;}- host[e]={(uint8_t*)t[0]->w,t[0]->scale,t[0]->fi_scale};host[count+e]={(uint8_t*)t[1]->w,t[1]->scale,t[1]->fi_scale};host[2*count+e]={(uint8_t*)t[2]->w,t[2]->scale,t[2]->fi_scale};+ host[e]={(uint8_t*)t[0]->w,t[0]->scale};host[count+e]={(uint8_t*)t[1]->w,t[1]->scale};host[2*count+e]={(uint8_t*)t[2]->w,t[2]->scale};}Dsv4CudaExpertSet *set=(Dsv4CudaExpertSet*)calloc(1,sizeof(*set));if(!set){free(host);return nullptr;}set->count=count;set->device=device;set->H=H;set->I=I;set->sg=sg;set->su=su;set->sd=sd;@@ -1559,18 +1392,10 @@ extern "C" int dsv4_cuda_route_moe(const Dsv4CudaActivation*input,Dsv4CudaTensor#endiffp8_sim<<<(H+127)/128,128,0,c->stream>>>(c->dx,1,H);build_moe_desc<<<1,K,0,c->stream>>>(c->mvdesc,set->table,c->expert_ids,c->dx,c->p1,c->p2,c->p3,c->p4,K,set->count,H,I);-#ifdef COLI_DSV4_FLASHINFER- if(!dsv4_flashinfer_grouped(c->mvdesc,2*K,I,H,input->device,c->stream))return 0;-#else- run_fp4_grouped(c->mvdesc,2*K,I,H,c->stream);-#endif+run_fp4_grouped(c->mvdesc,2*K,I,H,c->stream);expert_act_grouped<<<(K*I+255)/256,256,0,c->stream>>>(c->p3,c->p1,c->p2,c->expert_weights,limit,K,I);fp8_sim<<<K*((I+127)/128),128,0,c->stream>>>(c->p3,K,I);-#ifdef COLI_DSV4_FLASHINFER- if(!dsv4_flashinfer_grouped(c->mvdesc+2*K,K,H,I,input->device,c->stream))return 0;-#else- run_fp4_grouped(c->mvdesc+2*K,K,H,I,c->stream);-#endif+run_fp4_grouped(c->mvdesc+2*K,K,H,I,c->stream);expert_reduce<<<(H+255)/256,256,0,c->stream>>>(output->data,c->p4,K,H);run_mv<8>((uint8_t*)set->sg->w,set->sg->scale,c->dx,c->p1,I,H,1,c->stream);run_mv<8>((uint8_t*)set->su->w,set->su->scale,c->dx,c->p2,I,H,1,c->stream);@@ -1715,5 +1540,5 @@ extern "C" int dsv4_cuda_wo(Dsv4CudaTensor *wa,Dsv4CudaTensor *wb,int groups,flobf16_round<<<(H+255)/256,256,0,c->stream>>>(c->p2,H);return ok(cudaGetLastError(),"wo launch")&&ok(cudaMemcpyAsync(out,c->p2,hb,cudaMemcpyDeviceToHost,c->stream),"wo result download")&&ok(cudaStreamSynchronize(c->stream),"wo sync");}-extern "C" void dsv4_cuda_tensor_free(Dsv4CudaTensor*t){if(!t)return;cudaSetDevice(t->device);if(t->own_w&&t->w)cudaFree(t->w);if(t->bf16_cache)cudaFree(t->bf16_cache);if(t->own_scale&&t->scale)cudaFree(t->scale);if(t->own_dg_scale&&t->dg_scale)cudaFree(t->dg_scale);if(t->fi_scale)cudaFree(t->fi_scale);free(t);}+extern "C" void dsv4_cuda_tensor_free(Dsv4CudaTensor*t){if(!t)return;cudaSetDevice(t->device);if(t->own_w&&t->w)cudaFree(t->w);if(t->bf16_cache)cudaFree(t->bf16_cache);if(t->own_scale&&t->scale)cudaFree(t->scale);if(t->own_dg_scale&&t->dg_scale)cudaFree(t->dg_scale);free(t);}extern "C" long long dsv4_cuda_tensor_bytes(const Dsv4CudaTensor*t){return t?t->bytes:0;}diff --git a/c/backend_cuda_dsv4_flashinfer.cu b/c/backend_cuda_dsv4_flashinfer.cudeleted file mode 100644index 2b7e5e8e..00000000--- a/c/backend_cuda_dsv4_flashinfer.cu+++ /dev/null@@ -1,71 +0,0 @@-#include "backend_cuda_dsv4_flashinfer.h"-#include <cuda_bf16.h>-#include <cuda_fp8.h>-#include <flashinfer/gemm/group_gemm_mxfp4_groupwise_sm120.cuh>-#include <flashinfer/attention/sparse_mla_sm120/model/kv_cache_traits.cuh>-#include <cstdio>-#include <cstdlib>--namespace flashinfer::group_gemm {-template <bool SwapAB,int ScaleGranularity,typename ScaleConfig,typename ElementA,typename ElementB,- typename ElementSFA,typename ElementSFB,typename ElementD,typename ProblemShape,typename StrideA,- typename StrideB,typename StrideD,typename LayoutSFA,typename LayoutSFB>-__global__ void compute_pointer_group_gemm_args(- ElementA *packed_a,ElementB *B,ElementSFA *packed_sfa,ElementSFB *SFB,ElementD *D,int *m_indptr,- int n,int k,int num_groups,ProblemShape *problem_sizes,const ElementA **A_ptr,const ElementB **B_ptr,- const ElementSFA **SFA_ptr,const ElementSFB **SFB_ptr,ElementD **D_ptr,StrideA *stride_A,- StrideB *stride_B,StrideD *stride_D,LayoutSFA *layout_SFA,LayoutSFB *layout_SFB){- int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=num_groups)return;- auto A_list=reinterpret_cast<const ElementA *const *>(packed_a);- auto SFA_list=reinterpret_cast<const ElementSFA *const *>(packed_sfa);- constexpr size_t mn_align=128,k_align=(size_t)ScaleGranularity*4;- size_t sf_n=((size_t)n+mn_align-1)/mn_align*mn_align,swk=((size_t)k+k_align-1)/k_align*k_align,sf_k=swk/ScaleGranularity;- asm volatile("griddepcontrol.wait;");asm volatile("griddepcontrol.launch_dependents;");- int mo=m_indptr[i],mn=m_indptr[i+1];size_t m=(size_t)(mn-mo),sf_mo=((size_t)mo+(size_t)i*(mn_align-1))/mn_align*mn_align;- static_assert(SwapAB,"pointer grouped GEMM is decode-only");- problem_sizes[i]=ProblemShape(n,m,k);stride_A[i]=cutlass::make_cute_packed_stride(StrideA{},{n,k,1});- stride_B[i]=cutlass::make_cute_packed_stride(StrideB{},{m,k,1});stride_D[i]=cutlass::make_cute_packed_stride(StrideD{},{n,m,1});- A_ptr[i]=A_list[i];B_ptr[i]=B+(size_t)mo*k;D_ptr[i]=D+(size_t)mo*n;- layout_SFA[i]=ScaleConfig::tile_atom_to_shape_SFA(make_shape((int)sf_n,(int)m,(int)swk,1));SFA_ptr[i]=SFA_list[i];- layout_SFB[i]=ScaleConfig::tile_atom_to_shape_SFB(make_shape((int)sf_n,(int)m,(int)swk,1));SFB_ptr[i]=SFB+sf_mo*sf_k;-}-#define compute_sm120_cutlass_group_gemm_args compute_pointer_group_gemm_args-INSTANTIATE_GROUP_GEMM_MXFP4_GROUPWISE_SM120(- 128,32,128,true,cutlass::float_e4m3_t,cutlass::float_e2m1_t,- cutlass::float_ue8m0_t,cutlass::float_ue8m0_t,cutlass::bfloat16_t,- fp8,fp4,ue8m0,ue8m0,bf16)-#undef compute_sm120_cutlass_group_gemm_args-}-using namespace flashinfer::group_gemm;--namespace flashinfer::sparse_mla_sm120 {-bool launch_sparse_mla_decode_dsv4(ModelType,int,int,int,int,int,const bf16*,const uint8_t*,- const int32_t*,bf16*,float*,bf16*,float*,const int*,- const float*,const uint8_t*,const int32_t*,const int*,int,- int,size_t,int,float,size_t,cudaStream_t);-}--struct Desc { const uint8_t *w,*scale;const float *x;float *y; };-struct FiCtx { int device=-1;const uint8_t **wp=nullptr,**sp=nullptr;uint8_t *ws=nullptr,*a=nullptr,*as=nullptr;__nv_bfloat16 *out=nullptr;int *indptr=nullptr;void *iw=nullptr,*fw=nullptr;cudaGraph_t graph=nullptr;cudaGraphExec_t exec=nullptr;size_t wpc=0,spc=0,wsc=0,ac=0,asc=0,oc=0,ic=0; };-static FiCtx fc[16][2];-static int grow(void **p,size_t *cap,size_t n){if(*cap>=n)return 1;if(*p)cudaFree(*p);*p=nullptr;*cap=0;if(cudaMalloc(p,n)!=cudaSuccess)return 0;*cap=n;return 1;}-__device__ __forceinline__ int sf_off(int row,int col,int cols){return (col&3)+(col/4)*512+(row&31)*16+((row&127)/32)*4+(row/128)*128*((cols+3)&~3);}-__global__ void stage_pointer(const Desc*d,const uint8_t**wp,const uint8_t**sp,int count){int t=threadIdx.x;if(t<count){wp[t]=d[t].w;sp[t]=d[t].scale;}}-__global__ void quant_activation(const Desc*d,uint8_t*a,uint8_t*s,int count,int I){int g=blockIdx.x/(I/32),b=blockIdx.x%(I/32),i=b*32+threadIdx.x;float v=d[g].x[i],mx=fabsf(v);for(int m=16;m;m>>=1)mx=fmaxf(mx,__shfl_xor_sync(0xffffffff,mx,m));float raw=fmaxf(mx,1e-4f)/448.f;int e;frexpf(raw,&e);float sc=ldexpf(1.f,e-1);if(sc<raw)sc*=2;int se;frexpf(sc,&se);uint8_t q=__nv_cvt_float_to_fp8(v/sc,__NV_SATFINITE,__NV_E4M3);for(int r=0;r<4;r++){a[((long long)g*4+r)*I+i]=q;if(!threadIdx.x)s[(long long)g*128*((I/32+3)&~3)+sf_off(r,b,I/32)]=(uint8_t)(se-1+127);}}-__global__ void make_indptr(int*p,int count){int i=threadIdx.x;if(i<=count)p[i]=i*4;}-__global__ void extract(const Desc*d,const __nv_bfloat16*out,int count,int O){int p=blockIdx.x*blockDim.x+threadIdx.x;if(p<count*O){int g=p/O,o=p%O;d[g].y[o]=__bfloat162float(out[(long long)g*4*O+o]);}}--extern "C" int dsv4_flashinfer_grouped(const void *descriptors,int count,int O,int I,int device,cudaStream_t stream){if(!descriptors||count<1||count>16||O%128||I%128||device<0||device>=16)return 0;if(cudaSetDevice(device)!=cudaSuccess)return 0;FiCtx&c=fc[device][O>I];c.device=device;size_t sb=(size_t)count*O*(I/32),ab=(size_t)count*4*I,asb=(size_t)count*128*((I/32+3)&~3),ob=(size_t)count*4*O*2;- if(!grow((void**)&c.wp,&c.wpc,count*sizeof(*c.wp))||!grow((void**)&c.sp,&c.spc,count*sizeof(*c.sp))||!grow((void**)&c.ws,&c.wsc,sb)||!grow((void**)&c.a,&c.ac,ab)||!grow((void**)&c.as,&c.asc,asb)||!grow((void**)&c.out,&c.oc,ob)||!grow((void**)&c.indptr,&c.ic,(count+1)*4)||(!c.iw&&cudaMalloc(&c.iw,8<<20)!=cudaSuccess)||(!c.fw&&cudaMalloc(&c.fw,64<<20)!=cudaSuccess))return 0;- if(c.exec)return cudaGraphLaunch(c.exec,stream)==cudaSuccess;- auto launch=[&]()->int{cudaMemsetAsync(c.as,0,asb,stream);stage_pointer<<<1,32,0,stream>>>((const Desc*)descriptors,c.wp,c.sp,count);quant_activation<<<count*(I/32),32,0,stream>>>((const Desc*)descriptors,c.a,c.as,count,I);make_indptr<<<1,32,0,stream>>>(c.indptr,count);cudaError_t e=CutlassMXFP4GroupwiseScaledGroupGEMMSM120<128,32,128,true,cutlass::float_e4m3_t,cutlass::float_e2m1_t,cutlass::float_ue8m0_t,cutlass::float_ue8m0_t,cutlass::bfloat16_t>(c.iw,8<<20,c.fw,64<<20,(cutlass::float_e4m3_t*)c.a,(cutlass::float_e2m1_t*)c.wp,(cutlass::float_ue8m0_t*)c.as,(cutlass::float_ue8m0_t*)c.sp,(cutlass::bfloat16_t*)c.out,c.indptr,O,I,count,stream,device);if(e!=cudaSuccess)return 0;extract<<<(count*O+255)/256,256,0,stream>>>((const Desc*)descriptors,c.out,count,O);return cudaGetLastError()==cudaSuccess;};- static int graph=-1;if(graph<0){const char*p=getenv("DSV4_CUDA_FLASHINFER_GRAPH");graph=p&&atoi(p)!=0;}if(!graph)return launch();if(c.exec)return cudaGraphLaunch(c.exec,stream)==cudaSuccess;if(!launch()||cudaStreamSynchronize(stream)!=cudaSuccess||cudaStreamBeginCapture(stream,cudaStreamCaptureModeThreadLocal)!=cudaSuccess||!launch()||cudaStreamEndCapture(stream,&c.graph)!=cudaSuccess||cudaGraphInstantiate(&c.exec,c.graph,0)!=cudaSuccess)return 0;return 1;}-extern "C" int dsv4_flashinfer_sparse_mla(const void*q,const void*kv,const int*indices,void*mid,float*mid_lse,void*out,float*out_lse,const int*length,const float*sink,const void*extra_kv,const int*extra_indices,const int*extra_length,int extra_topk,int pbs_extra,size_t extra_stride,int heads,int topk,int tokens,int splits,int chunks,float scale,size_t stride,cudaStream_t stream){- using namespace flashinfer::sparse_mla_sm120;- if(!q||!kv||!indices||!mid||!mid_lse||!out||!length||heads<1||tokens<1||splits<1)return 0;- return launch_sparse_mla_decode_dsv4(ModelType::DSV4,heads,topk,64,tokens,splits,- (const bf16*)q,(const uint8_t*)kv,(const int32_t*)indices,(bf16*)mid,mid_lse,- (bf16*)out,out_lse,length,sink,(const uint8_t*)extra_kv,(const int32_t*)extra_indices,- extra_length,extra_topk,pbs_extra,extra_stride,chunks,scale,stride,stream);-}-extern "C" void dsv4_flashinfer_shutdown(void){for(auto&dev:fc)for(auto&c:dev)if(c.device>=0){cudaSetDevice(c.device);if(c.exec)cudaGraphExecDestroy(c.exec);if(c.graph)cudaGraphDestroy(c.graph);cudaFree(c.wp);cudaFree(c.sp);cudaFree(c.ws);cudaFree(c.a);cudaFree(c.as);cudaFree(c.out);cudaFree(c.indptr);cudaFree(c.iw);cudaFree(c.fw);c=FiCtx{};}}diff --git a/c/backend_cuda_dsv4_flashinfer.h b/c/backend_cuda_dsv4_flashinfer.hdeleted file mode 100644index fd66632f..00000000--- a/c/backend_cuda_dsv4_flashinfer.h+++ /dev/null@@ -1,20 +0,0 @@-#ifndef COLIBRI_BACKEND_CUDA_DSV4_FLASHINFER_H-#define COLIBRI_BACKEND_CUDA_DSV4_FLASHINFER_H-#include <cuda_runtime.h>-#ifdef __cplusplus-extern "C" {-#endif-int dsv4_flashinfer_grouped(const void *descriptors,int count,int O,int I,int device,cudaStream_t stream);-int dsv4_flashinfer_sparse_mla(const void *q,const void *kv_cache,const int *indices,- void *mid_out,float *mid_lse,void *output,float *out_lse,- const int *topk_length,const float *attn_sink,- const void *extra_kv_cache,const int *extra_indices,- const int *extra_topk_length,int extra_topk,int pbs_extra,- size_t stride_extra_kv_block,int heads,int topk,int tokens,- int splits,int chunks_per_block,float sm_scale,- size_t stride_kv_block,cudaStream_t stream);-void dsv4_flashinfer_shutdown(void);-#ifdef __cplusplus-}-#endif-#endifdiff --git a/c/backend_cuda_dsv4_mhc_vllm.cu b/c/backend_cuda_dsv4_mhc_vllm.cudeleted file mode 100644index 1a1e1ed8..00000000--- a/c/backend_cuda_dsv4_mhc_vllm.cu+++ /dev/null@@ -1,535 +0,0 @@-// SPDX-License-Identifier: Apache-2.0-// Generated by vLLM TileLang at ffd46bfab2128bb84146050e98b51a617c6575ab-#include <tl_templates/cuda/gemm.h>-#include <tl_templates/cuda/copy.h>-#include <tl_templates/cuda/reduce.h>-#include <tl_templates/cuda/ldsm.h>-#include <tl_templates/cuda/threadblock_swizzle.h>-#include <tl_templates/cuda/debug.h>-#ifdef ENABLE_BF16-#include <tl_templates/cuda/cuda_bf16_fallbacks.cuh>-#endif--extern "C" __global__ void mhc_fused_tilelang_kernel(const float* comb_mix, const float* post_mix, const bfloat16_t* residual_in, bfloat16_t* residual_out, float* rp_out, const float* weight_t, const bfloat16_t* x_in, float* yp_out, int num_tokens, int split_k);-extern "C" __global__ void __launch_bounds__(256, 1) mhc_fused_tilelang_kernel(const float* comb_mix, const float* post_mix, const bfloat16_t* residual_in, bfloat16_t* residual_out, float* rp_out, const float* weight_t, const bfloat16_t* x_in, float* yp_out, int num_tokens, int split_k) {- extern __shared__ __align__(1024) uchar buf_dyn_shmem[];- float acc[2];- float sqr[1];- float pm[4];- float cm[16];- float new_r[4];- float v = 0x0p+0f/*0.000000e+00*/;- float v2 = 0x0p+0f/*0.000000e+00*/;- float broadcast_var = 0x0p+0f/*0.000000e+00*/;- *(float2*)(acc + 0) = make_float2(broadcast_var, broadcast_var);- sqr[0] = 0x0p+0f/*0.000000e+00*/;- cudaGridDependencySynchronize();- if (((int)threadIdx.x) < 4) {- ((float*)buf_dyn_shmem)[((int)threadIdx.x)] = post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))];- }- if (((int)threadIdx.x) < 16) {- ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] = comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + ((int64_t)((int)threadIdx.x)))];- }- __syncthreads();- #pragma unroll- for (int j = 0; j < 4; ++j) {- pm[j] = ((float*)buf_dyn_shmem)[j];- }- #pragma unroll- for (int j_1 = 0; j_1 < 4; ++j_1) {- #pragma unroll- for (int k = 0; k < 4; ++k) {- cm[((k * 4) + j_1)] = ((float*)buf_dyn_shmem)[(((k * 4) + j_1) + 4)];- }- }- for (int it = 0; it < ((4096 / split_k) >> 8); ++it) {- #pragma unroll- for (int j_2 = 0; j_2 < 4; ++j_2) {- new_r[j_2] = (pm[j_2] * ((float)x_in[((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))]));- #pragma unroll- for (int k_1 = 0; k_1 < 4; ++k_1) {- new_r[j_2] = (new_r[j_2] + (cm[((k_1 * 4) + j_2)] * ((float)residual_in[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)k_1) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))])));- }- }- if (((int)blockIdx.y) == 0) {- #pragma unroll- for (int j_3 = 0; j_3 < 4; ++j_3) {- residual_out[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)j_3) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))] = ((bfloat16_t)new_r[j_3]);- sqr[0] = (sqr[0] + (new_r[j_3] * new_r[j_3]));- }- }- #pragma unroll- for (int n = 0; n < 2; ++n) {- #pragma unroll- for (int j_4 = 0; j_4 < 4; ++j_4) {- acc[n] = (acc[n] + (weight_t[((((((((int64_t)((int)blockIdx.y)) * (int64_t)32768) + (((int64_t)n) * (int64_t)16384)) + (((int64_t)j_4) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))] * new_r[j_4]));- }- }- }- #pragma unroll- for (int n_1 = 0; n_1 < 2; ++n_1) {- acc[n_1] = tl::warp_reduce_sum(acc[n_1]);- }- if (((int)blockIdx.y) == 0) {- sqr[0] = tl::warp_reduce_sum(sqr[0]);- }- if ((((int)threadIdx.x) % 32) == 0) {- #pragma unroll- for (int n_2 = 0; n_2 < 2; ++n_2) {- ((float*)buf_dyn_shmem)[((((((int)threadIdx.x) >> 5) * 3) + n_2) + 20)] = acc[n_2];- }- if (((int)blockIdx.y) == 0) {- ((float*)buf_dyn_shmem)[(((((int)threadIdx.x) >> 5) * 3) + 22)] = sqr[0];- }- }- __syncthreads();- if ((((int)threadIdx.x) >> 5) == 0) {- if ((((int)threadIdx.x) & 31) < 2) {- #pragma unroll- for (int w = 0; w < 8; ++w) {- v = (v + ((float*)buf_dyn_shmem)[(((w * 3) + (((int)threadIdx.x) & 31)) + 20)]);- }- yp_out[((((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)((int)blockIdx.z)) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)blockIdx.y)) * (int64_t)2)) + (((int64_t)((int)threadIdx.x)) & (int64_t)31))] = v;- }- if ((((int)blockIdx.y) == 0) && ((((int)threadIdx.x) % 32) == 0)) {- #pragma unroll- for (int w_1 = 0; w_1 < 8; ++w_1) {- v2 = (v2 + ((float*)buf_dyn_shmem)[((w_1 * 3) + 22)]);- }- rp_out[((((int64_t)((int)blockIdx.z)) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))] = v2;- }- }-cudaTriggerProgrammaticLaunchCompletion();-}--extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(- float *, const float *, const float *, const float *, const float *,- bfloat16_t *, const bfloat16_t *, float *, const bfloat16_t *, int);--extern "C" cudaError_t dsv4_vllm_mhc_post_pre_norm_batch(- cudaStream_t stream, const float *comb_mix, const void *residual_in,- const float *post_mix, const void *x_in, const float *weight_t,- float *gemm_out_mul, float *gemm_out_sqrsum, void *residual_out,- const float *hc_scale, const float *hc_base, void *layer_input,- const void *norm_weight, int num_tokens) {- if (num_tokens < 1) return cudaErrorInvalidValue;- const int split_k = 8;- mhc_fused_tilelang_kernel<<<dim3(num_tokens, 12, split_k), 256, 176, stream>>>(- comb_mix, post_mix, static_cast<const bfloat16_t *>(residual_in),- static_cast<bfloat16_t *>(residual_out), gemm_out_sqrsum, weight_t,- static_cast<const bfloat16_t *>(x_in), gemm_out_mul, num_tokens,- split_k);- cudaError_t error = cudaGetLastError();- if (error != cudaSuccess) return error;- mhc_pre_big_fuse_with_norm_tilelang_kernel<<<num_tokens, 96, 29040, stream>>>(- const_cast<float *>(comb_mix), gemm_out_mul, gemm_out_sqrsum, hc_base,- hc_scale, static_cast<bfloat16_t *>(layer_input),- static_cast<const bfloat16_t *>(norm_weight),- const_cast<float *>(post_mix),- static_cast<const bfloat16_t *>(residual_out), num_tokens);- return cudaGetLastError();-}--extern "C" cudaError_t dsv4_vllm_mhc_post_pre_norm(- cudaStream_t stream, const float *comb_mix, const void *residual_in,- const float *post_mix, const void *x_in, const float *weight_t,- float *gemm_out_mul, float *gemm_out_sqrsum, void *residual_out,- const float *hc_scale, const float *hc_base, void *layer_input,- const void *norm_weight) {- return dsv4_vllm_mhc_post_pre_norm_batch(- stream, comb_mix, residual_in, post_mix, x_in, weight_t, gemm_out_mul,- gemm_out_sqrsum, residual_out, hc_scale, hc_base, layer_input,- norm_weight, 1);-}---#include <math_constants.h>-#include <tl_templates/cuda/gemm.h>-#include <tl_templates/cuda/copy.h>-#include <tl_templates/cuda/reduce.h>-#include <tl_templates/cuda/ldsm.h>-#include <tl_templates/cuda/threadblock_swizzle.h>-#include <tl_templates/cuda/debug.h>-#ifdef ENABLE_BF16-#include <tl_templates/cuda/cuda_bf16_fallbacks.cuh>-#endif--extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens);-extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) {- extern __shared__ __align__(1024) uchar buf_dyn_shmem[];- float mixes[1];- float rms[1];- float cm[1];- float row_max[1];- float row_sum[1];- float col_sum[1];- float sumsq_per_pos[16];- float sumsq[1];- float rsqrt_norm[1];- float xl[64];- float ol[16];- bfloat16_t xs_local_cast[8];- bfloat16_t xs_local_cast_1[8];- bfloat16_t xs_local_cast_2[8];- float w_local[16];- float ol_1[16];- bfloat16_t w_shared_local_cast_3[8];- bfloat16_t output_shared_local_cast_4[8];- bfloat16_t layer_input_local_cast_5[8];- bfloat16_t w_shared_local_cast_6[8];- bfloat16_t output_shared_local_cast_7[8];- bfloat16_t layer_input_local_cast_8[8];- bfloat16_t w_shared_local_cast_9[8];- bfloat16_t output_shared_local_cast_10[8];- bfloat16_t layer_input_local_cast_11[8];- mixes[0] = 0x0p+0f/*0.000000e+00*/;- rms[0] = 0x0p+0f/*0.000000e+00*/;- cudaGridDependencySynchronize();- for (int i_split = 0; i_split < 8; ++i_split) {- rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]);- }- rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));- mixes[0] = 0x0p+0f/*0.000000e+00*/;- for (int i_split_1 = 0; i_split_1 < 8; ++i_split_1) {- mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]);- }- mixes[0] = (mixes[0] * rms[0]);- if ((((int)threadIdx.x) / 24) == 0) {- ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0];- }- __syncthreads();- if (((int)threadIdx.x) < 32) {- if (((int)threadIdx.x) < 4) {- post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/);- }- cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]);- row_max[0] = -CUDART_INF_F;- row_max[0] = max(row_max[0], cm[0]);- row_max[0] = tl::AllReduce<tl::MaxOp, 4, 1, 0, tl::NamedBarrier<32>>::run(row_max[0]);- cm[0] = expf((cm[0] - row_max[0]));- row_sum[0] = 0x0p+0f/*0.000000e+00*/;- row_sum[0] = (row_sum[0] + cm[0]);- row_sum[0] = tl::AllReduce<tl::SumOp, 4, 1, 0, tl::NamedBarrier<32>>::run(row_sum[0]);- cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/);- col_sum[0] = 0x0p+0f/*0.000000e+00*/;- col_sum[0] = (col_sum[0] + cm[0]);- col_sum[0] = tl::AllReduce<tl::SumOp, 16, 4, 0, tl::NamedBarrier<32>>::run(col_sum[0]);- cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));- for (int __1 = 0; __1 < 19; ++__1) {- row_sum[0] = 0x0p+0f/*0.000000e+00*/;- row_sum[0] = (row_sum[0] + cm[0]);- row_sum[0] = tl::AllReduce<tl::SumOp, 4, 1, 0, tl::NamedBarrier<32>>::run(row_sum[0]);- cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));- col_sum[0] = 0x0p+0f/*0.000000e+00*/;- col_sum[0] = (col_sum[0] + cm[0]);- col_sum[0] = tl::AllReduce<tl::SumOp, 16, 4, 0, tl::NamedBarrier<32>>::run(col_sum[0]);- cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));- }- if ((((int)threadIdx.x) >> 4) == 0) {- comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0];- }- } else {- if (((int)threadIdx.x) < 36) {- ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/);- }- #pragma unroll- for (int i = 0; i < 4; ++i) {- float broadcast_var = 0x0p+0f/*0.000000e+00*/;- *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var);- }- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_1 = 0; i_1 < 8; ++i_1) {- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))])));- }- tl::cp_async_commit();- #pragma unroll- for (int i_2 = 0; i_2 < 8; ++i_2) {- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)])));- }- tl::cp_async_commit();- for (int i0_h = 0; i0_h < 2; ++i0_h) {- tl::cp_async_wait<1>();- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_3 = 0; i_3 < 8; ++i_3) {- *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208));- for (int vec = 0; vec < 2; ++vec) {- float4 __2;- uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4));- ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]);- ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]);- *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2;- }- }- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_4 = 0; i_4 < 8; ++i_4) {- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)])));- }- tl::cp_async_commit();- #pragma unroll- for (int i_5 = 0; i_5 < 4; ++i_5) {- float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/;- *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1);- }- tl::__sync_thread_partial<3, 64>();- for (int i_hc = 0; i_hc < 4; ++i_hc) {- float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)];- #pragma unroll- for (int i_6 = 0; i_6 < 16; ++i_6) {- ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)]));- }- }- #pragma unroll- for (int i_7 = 0; i_7 < 4; ++i_7) {- float4 __3;- float4 v__1 = *(float4*)(ol + (i_7 * 4));- float4 v__2 = *(float4*)(sumsq_per_pos + (i_7 * 4));- *(float2*)(&(__3.x)) = tl::fma2(*(float2*)(&(v__1.x)), *(float2*)(&(v__1.x)), *(float2*)(&(v__2.x)));- *(float2*)(&(__3.z)) = tl::fma2(*(float2*)(&(v__1.z)), *(float2*)(&(v__1.z)), *(float2*)(&(v__2.z)));- *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3;- uint2 __4;- float4 v__3 = *(float4*)(ol + (i_7 * 4));- (reinterpret_cast<__nv_bfloat162*>(&__4))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]);- (reinterpret_cast<__nv_bfloat162*>(&__4))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]);- *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __4;- }- }- tl::cp_async_wait<1>();- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_8 = 0; i_8 < 8; ++i_8) {- *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208));- for (int vec_1 = 0; vec_1 < 2; ++vec_1) {- float4 __5;- uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4));- ((float2*)(&__5))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]);- ((float2*)(&__5))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]);- *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __5;- }- }- #pragma unroll- for (int i_9 = 0; i_9 < 4; ++i_9) {- float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/;- *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2);- }- for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) {- float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)];- #pragma unroll- for (int i_10 = 0; i_10 < 16; ++i_10) {- ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)]));- }- }- #pragma unroll- for (int i_11 = 0; i_11 < 4; ++i_11) {- float4 __6;- float4 v__5 = *(float4*)(ol + (i_11 * 4));- float4 v__6 = *(float4*)(sumsq_per_pos + (i_11 * 4));- *(float2*)(&(__6.x)) = tl::fma2(*(float2*)(&(v__5.x)), *(float2*)(&(v__5.x)), *(float2*)(&(v__6.x)));- *(float2*)(&(__6.z)) = tl::fma2(*(float2*)(&(v__5.z)), *(float2*)(&(v__5.z)), *(float2*)(&(v__6.z)));- *(float4*)(sumsq_per_pos + (i_11 * 4)) = __6;- uint2 __7;- float4 v__7 = *(float4*)(ol + (i_11 * 4));- (reinterpret_cast<__nv_bfloat162*>(&__7))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]);- (reinterpret_cast<__nv_bfloat162*>(&__7))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]);- *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __7;- }- tl::cp_async_wait<0>();- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_12 = 0; i_12 < 8; ++i_12) {- *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888));- for (int vec_2 = 0; vec_2 < 2; ++vec_2) {- float4 __8;- uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4));- ((float2*)(&__8))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]);- ((float2*)(&__8))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]);- *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __8;- }- }- #pragma unroll- for (int i_13 = 0; i_13 < 4; ++i_13) {- float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/;- *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3);- }- for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) {- float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)];- #pragma unroll- for (int i_14 = 0; i_14 < 16; ++i_14) {- ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)]));- }- }- #pragma unroll- for (int i_15 = 0; i_15 < 4; ++i_15) {- float4 __9;- float4 v__9 = *(float4*)(ol + (i_15 * 4));- float4 v__10 = *(float4*)(sumsq_per_pos + (i_15 * 4));- *(float2*)(&(__9.x)) = tl::fma2(*(float2*)(&(v__9.x)), *(float2*)(&(v__9.x)), *(float2*)(&(v__10.x)));- *(float2*)(&(__9.z)) = tl::fma2(*(float2*)(&(v__9.z)), *(float2*)(&(v__9.z)), *(float2*)(&(v__10.z)));- *(float4*)(sumsq_per_pos + (i_15 * 4)) = __9;- uint2 __10;- float4 v__11 = *(float4*)(ol + (i_15 * 4));- (reinterpret_cast<__nv_bfloat162*>(&__10))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]);- (reinterpret_cast<__nv_bfloat162*>(&__10))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]);- *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __10;- }- sumsq[0] = 0x0p+0f/*0.000000e+00*/;- #pragma unroll- for (int rv = 0; rv < 16; ++rv) {- sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]);- }- tl::__sync_thread_partial<3, 64>();- sumsq[0] = tl::AllReduce<tl::SumOp, 64, 1, 32, tl::NamedBarrier<64>>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192])));- rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));- #pragma unroll- for (int i_16 = 0; i_16 < 2; ++i_16) {- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)])));- }- tl::cp_async_commit();- #pragma unroll- for (int i_17 = 0; i_17 < 2; ++i_17) {- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)])));- }- tl::cp_async_commit();- for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) {- tl::cp_async_wait<1>();- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_18 = 0; i_18 < 2; ++i_18) {- *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080));- for (int vec_3 = 0; vec_3 < 2; ++vec_3) {- float4 __11;- uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4));- ((float2*)(&__11))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]);- ((float2*)(&__11))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]);- *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __11;- }- }- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_19 = 0; i_19 < 2; ++i_19) {- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)])));- }- tl::cp_async_commit();- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_20 = 0; i_20 < 2; ++i_20) {- *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984));- for (int vec_4 = 0; vec_4 < 2; ++vec_4) {- float4 __12;- float4 __13;- float4 __14;- uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4));- ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]);- ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]);- float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]);- *(float2*)(&(__13.x)) = tl::mul2(*(float2*)(&(__14.x)), *(float2*)(&(v__14.x)));- *(float2*)(&(__13.z)) = tl::mul2(*(float2*)(&(__14.z)), *(float2*)(&(v__14.z)));- float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4)));- *(float2*)(&(__12.x)) = tl::mul2(*(float2*)(&(__13.x)), *(float2*)(&(v__15.x)));- *(float2*)(&(__12.z)) = tl::mul2(*(float2*)(&(__13.z)), *(float2*)(&(v__15.z)));- *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __12;- }- }- #pragma unroll- for (int i_21 = 0; i_21 < 2; ++i_21) {- for (int vec_5 = 0; vec_5 < 2; ++vec_5) {- uint2 __15;- float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4)));- (reinterpret_cast<__nv_bfloat162*>(&__15))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]);- (reinterpret_cast<__nv_bfloat162*>(&__15))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]);- *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __15;- }- *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0);- }- }- tl::cp_async_wait<1>();- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_22 = 0; i_22 < 2; ++i_22) {- *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080));- for (int vec_6 = 0; vec_6 < 2; ++vec_6) {- float4 __16;- uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4));- ((float2*)(&__16))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]);- ((float2*)(&__16))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]);- *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __16;- }- }- #pragma unroll- for (int i_23 = 0; i_23 < 2; ++i_23) {- *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032));- for (int vec_7 = 0; vec_7 < 2; ++vec_7) {- float4 __17;- float4 __18;- float4 __19;- uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4));- ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]);- ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]);- float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]);- *(float2*)(&(__18.x)) = tl::mul2(*(float2*)(&(__19.x)), *(float2*)(&(v__19.x)));- *(float2*)(&(__18.z)) = tl::mul2(*(float2*)(&(__19.z)), *(float2*)(&(v__19.z)));- float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4)));- *(float2*)(&(__17.x)) = tl::mul2(*(float2*)(&(__18.x)), *(float2*)(&(v__20.x)));- *(float2*)(&(__17.z)) = tl::mul2(*(float2*)(&(__18.z)), *(float2*)(&(v__20.z)));- *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __17;- }- }- #pragma unroll- for (int i_24 = 0; i_24 < 2; ++i_24) {- for (int vec_8 = 0; vec_8 < 2; ++vec_8) {- uint2 __20;- float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4)));- (reinterpret_cast<__nv_bfloat162*>(&__20))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]);- (reinterpret_cast<__nv_bfloat162*>(&__20))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]);- *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __20;- }- *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0);- }- tl::cp_async_wait<0>();- tl::__sync_thread_partial<3, 64>();- #pragma unroll- for (int i_25 = 0; i_25 < 2; ++i_25) {- *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104));- for (int vec_9 = 0; vec_9 < 2; ++vec_9) {- float4 __21;- uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4));- ((float2*)(&__21))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]);- ((float2*)(&__21))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]);- *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __21;- }- }- #pragma unroll- for (int i_26 = 0; i_26 < 2; ++i_26) {- *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056));- for (int vec_10 = 0; vec_10 < 2; ++vec_10) {- float4 __22;- float4 __23;- float4 __24;- uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4));- ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]);- ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]);- float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]);- *(float2*)(&(__23.x)) = tl::mul2(*(float2*)(&(__24.x)), *(float2*)(&(v__24.x)));- *(float2*)(&(__23.z)) = tl::mul2(*(float2*)(&(__24.z)), *(float2*)(&(v__24.z)));- float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4)));- *(float2*)(&(__22.x)) = tl::mul2(*(float2*)(&(__23.x)), *(float2*)(&(v__25.x)));- *(float2*)(&(__22.z)) = tl::mul2(*(float2*)(&(__23.z)), *(float2*)(&(v__25.z)));- *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __22;- }- }- #pragma unroll- for (int i_27 = 0; i_27 < 2; ++i_27) {- for (int vec_11 = 0; vec_11 < 2; ++vec_11) {- uint2 __25;- float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4)));- (reinterpret_cast<__nv_bfloat162*>(&__25))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]);- (reinterpret_cast<__nv_bfloat162*>(&__25))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]);- *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __25;- }- *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0);- }- }- cudaTriggerProgrammaticLaunchCompletion();-}diff --git a/c/backend_cuda_dsv4_mhc_vllm.h b/c/backend_cuda_dsv4_mhc_vllm.hdeleted file mode 100644index 02ab41d9..00000000--- a/c/backend_cuda_dsv4_mhc_vllm.h+++ /dev/null@@ -1,27 +0,0 @@-#ifndef COLIBRI_BACKEND_CUDA_DSV4_MHC_VLLM_H-#define COLIBRI_BACKEND_CUDA_DSV4_MHC_VLLM_H--#include <cuda_runtime_api.h>--#ifdef __cplusplus-extern "C" {-#endif--cudaError_t dsv4_vllm_mhc_post_pre_norm(- cudaStream_t stream, const float *comb_mix, const void *residual_in,- const float *post_mix, const void *x_in, const float *weight_t,- float *gemm_out_mul, float *gemm_out_sqrsum, void *residual_out,- const float *hc_scale, const float *hc_base, void *layer_input,- const void *norm_weight);-cudaError_t dsv4_vllm_mhc_post_pre_norm_batch(- cudaStream_t stream, const float *comb_mix, const void *residual_in,- const float *post_mix, const void *x_in, const float *weight_t,- float *gemm_out_mul, float *gemm_out_sqrsum, void *residual_out,- const float *hc_scale, const float *hc_base, void *layer_input,- const void *norm_weight, int num_tokens);--#ifdef __cplusplus-}-#endif--#endifdiff --git a/c/backend_cuda_dsv4_qkv_vllm.cu b/c/backend_cuda_dsv4_qkv_vllm.cudeleted file mode 100644index e43c90a0..00000000--- a/c/backend_cuda_dsv4_qkv_vllm.cu+++ /dev/null@@ -1,739 +0,0 @@-/*- * SPDX-License-Identifier: Apache-2.0- * SPDX-FileCopyrightText: Copyright contributors to the vLLM project- *- * Horizontally-fused DeepseekV4-MLA kernel:- * - Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last ROPE_DIM- * - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged- * cache insert- *- * Structured after `applyMLARopeAndAssignQKVKernelGeneration` in- * TensorRT-LLM's mlaKernels.cu: one kernel, one grid, with head-slot- * dispatch choosing Q vs KV work per warp. The per-warp RMSNorm/RoPE- * skeleton is adapted from vllm-deepseek_v4's existing- * `fusedQKNormRopeKernel` (csrc/fused_qknorm_rope_kernel.cu).- *- * Assumptions (hard-coded for DeepseekV4 attention):- * HEAD_DIM = 512- * ROPE_DIM = 64 (RoPE applied to dims [NOPE_DIM, HEAD_DIM))- * NOPE_DIM = 448- * QUANT_BLOCK = 64 (UE8M0 FP8 quant block)- * FP8_MAX = 224.0f on ROCm FNUZ / 448.0f on OCP- * is_neox=false (GPT-J interleaved pairs)- * cos_sin_cache layout [max_pos, rope_dim] = cos || sin (cos first, sin- * second along last dim; each half is rope_dim/2 = 32 values)- *- * Cache layout per paged-cache block (block_size tokens):- * [0, bs*576): token data, 448 fp8 + 128 bf16 each- * [bs*576, bs*576 + bs*8): UE8M0 scales, 7 real + 1 pad per token- */--#include <cmath>-#include <cuda_bf16.h>-#include <cuda_fp8.h>-#include <cuda_runtime.h>-#include <cstdint>-#include <type_traits>--namespace vllm {-template <typename T> struct _typeConvert;-template <> struct _typeConvert<__nv_bfloat16> {- using packed_hip_type = __nv_bfloat162;- __device__ static float2 convert(__nv_bfloat162 v) { return __bfloat1622float2(v); }- __device__ static __nv_bfloat162 convert(float2 v) { return __floats2bfloat162_rn(v.x, v.y); }- __device__ static __nv_bfloat16 convert(float v) { return __float2bfloat16(v); }- __device__ static float convert(__nv_bfloat16 v) { return __bfloat162float(v); }-};-}--#ifndef FINAL_MASK- #ifdef USE_ROCM- #define FINAL_MASK 0xffffffffffffffffULL- #else- #define FINAL_MASK 0xffffffffu- #endif-#endif--#ifdef USE_ROCM-// ROCm-compatible FP8 conversion helpers-__device__ __forceinline__ uint8_t rocm_cvt_float_to_fp8_e4m3(float val) {- // gfx942 uses FNUZ FP8; other ROCm targets use OCP E4M3.- #if defined(__gfx942__)- __hip_fp8_e4m3_fnuz fp8_val(val);- #else- __hip_fp8_e4m3 fp8_val(val);- #endif- return reinterpret_cast<uint8_t&>(fp8_val);-}-#endif--namespace vllm {-namespace deepseek_v4_fused_ops {--namespace {-inline int getSMVersion() {- int device = 0;- cudaDeviceProp props{};- cudaGetDevice(&device);- cudaGetDeviceProperties(&props, device);- return props.major * 10 + props.minor;-}-} // namespace--// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-// Constants-// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-constexpr int kHeadDim = 512;-constexpr int kRopeDim = 64;-constexpr int kNopeDim = kHeadDim - kRopeDim; // 448-constexpr int kQuantBlock = 64;-constexpr int kNumQuantBlocks = kNopeDim / kQuantBlock; // 7-constexpr int kScaleBytesPerToken = kNumQuantBlocks + 1; // 8 (7 real + 1 pad)-constexpr int kTokenDataBytes = kNopeDim + kRopeDim * 2; // 448 + 128 = 576-// FNUZ on gfx942 / OCP elsewhere. FNUZ uses 224.0 (not the dtype's raw-// 240.0) to match the rest of vLLM's FNUZ pipeline.-#if defined(USE_ROCM) && defined(__gfx942__)-constexpr float kFp8Max = 224.0f;-#else-constexpr float kFp8Max = 448.0f;-#endif--#ifndef USE_ROCM-// When num_tokens is less than this threshold,-// run the reduced grid variant on cuda-constexpr float NUM_TOKEN_CUTOFF = 1024;-#endif--// Per-warp layout: 32 lanes Γ 16 elems/lane = 512 elems = HEAD_DIM.-constexpr int kNumLanes = 32;-constexpr int kElemsPerLane = kHeadDim / kNumLanes; // 16--// Pack this lane's 16 fp32 elements into per-tensor E4M3 FP8 (one uint4 = 16-// B), scaling by `scale` (a reciprocal scale) and saturating to Β±448. Used by-// the FlashInfer full-cache path for both the Q and KV stores.-__device__ __forceinline__ uint4 packFp8E4M3x16(float const* values,- float const scale) {-#ifndef USE_ROCM- uint4 out;- auto* out2 = reinterpret_cast<__nv_fp8x2_storage_t*>(&out);- #pragma unroll- for (int i = 0; i < kElemsPerLane / 2; i++) {- float2 scaled =- make_float2(values[2 * i] * scale, values[2 * i + 1] * scale);- scaled.x = fminf(fmaxf(scaled.x, -kFp8Max), kFp8Max);- scaled.y = fminf(fmaxf(scaled.y, -kFp8Max), kFp8Max);- out2[i] = __nv_cvt_float2_to_fp8x2(scaled, __NV_SATFINITE, __NV_E4M3);- }- return out;-#else- uint8_t out_bytes[kElemsPerLane];- #pragma unroll- for (int i = 0; i < kElemsPerLane; i++) {- float scaled = values[i] * scale;- scaled = fminf(fmaxf(scaled, -kFp8Max), kFp8Max);- out_bytes[i] = rocm_cvt_float_to_fp8_e4m3(scaled);- }- return *reinterpret_cast<uint4 const*>(out_bytes);-#endif-}--// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-// Small inline helpers-// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-__device__ __forceinline__ float warp4MaxAbs(float val) {- // Reduce absolute max across 4 consecutive lanes (lane id & 3 group).- float peer = __shfl_xor_sync(FINAL_MASK, val, 1);- val = fmaxf(val, peer);- peer = __shfl_xor_sync(FINAL_MASK, val, 2);- val = fmaxf(val, peer);- return val;-}--template <typename T>-__device__ __forceinline__ float warpSum(float val) {-#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- val += __shfl_xor_sync(FINAL_MASK, val, mask, 32);- }- return val;-}--// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-// Per-slot inner pipeline-// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-// Shared by both kernel variants: 1 CTA per (token, head) pair vs. 1 CTA per-// token. Templated on `kNumHeadsQPadded` so the KV-sentinel comparison and-// q_out stride fold to compile-time constants.-//-// Slot layout (per token):-// slot < num_heads_q β live-Q (RMSNorm + RoPE,-// read q_in β-// write q_out)-// num_heads_q <= slot < kNumHeadsQPadded β pad-Q (zero-fill q_out;-// v0/v1 unused)-// slot == kNumHeadsQPadded β KV (RoPE + UE8M0 quant-// + paged-cache-// insert)-template <typename scalar_t_in, int kNumHeadsQPadded>-__device__ __forceinline__ void processDeepseekV4Slot(- uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx,- int const dim_base, int const laneId, int const num_heads_q,- float const eps, scalar_t_in* __restrict__ q_out,- uint8_t* __restrict__ k_cache, int64_t const* __restrict__ slot_mapping,- int64_t const* __restrict__ position_ids,- float const* __restrict__ cos_sin_cache, int const cache_block_size,- int const kv_block_stride) {- using Converter = vllm::_typeConvert<scalar_t_in>;- bool const isKV = (slotIdx == kNumHeadsQPadded);- bool const isPadQ = !isKV && (slotIdx >= num_heads_q);-- // ββ Pad-Q branch: write 32 B of zeros and exit. βββββββββββββββββββββββββ- // FlashMLA reads these slots; bf16 +0.0 is bit pattern 0x0000, so a uint4- // zero literal is correct. Matches the live-Q branch's vectorized store.- if (isPadQ) {- scalar_t_in* dst =- q_out +- (static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *- kHeadDim +- dim_base;- uint4 const zero4 = {0u, 0u, 0u, 0u};- *reinterpret_cast<uint4*>(dst) = zero4;- *reinterpret_cast<uint4*>(dst + 8) = zero4;- return;- }-- // ββ Decode the bf16 β 16 fp32 registers βββββββββββββββββββββββββββββ- float elements[kElemsPerLane];- {- typename Converter::packed_hip_type const* p0 =- reinterpret_cast<typename Converter::packed_hip_type const*>(&v0);- typename Converter::packed_hip_type const* p1 =- reinterpret_cast<typename Converter::packed_hip_type const*>(&v1);-#pragma unroll- for (int i = 0; i < 4; i++) {- float2 f2 = Converter::convert(p0[i]);- elements[2 * i] = f2.x;- elements[2 * i + 1] = f2.y;- }-#pragma unroll- for (int i = 0; i < 4; i++) {- float2 f2 = Converter::convert(p1[i]);- elements[8 + 2 * i] = f2.x;- elements[8 + 2 * i + 1] = f2.y;- }- }-- // ββ Q branch: RMSNorm (no weight) βββββββββββββββββββββββββββββββββββ- if (!isKV) {- float sumOfSquares = 0.0f;-#pragma unroll- for (int i = 0; i < kElemsPerLane; i++) {- sumOfSquares += elements[i] * elements[i];- }- sumOfSquares = warpSum<float>(sumOfSquares);- float const rms_rcp =- rsqrtf(sumOfSquares / static_cast<float>(kHeadDim) + eps);-#pragma unroll- for (int i = 0; i < kElemsPerLane; i++) {- elements[i] = elements[i] * rms_rcp;- }- }-- // ββ GPT-J RoPE on dims [NOPE_DIM, HEAD_DIM) βββββββββββββββββββββββββββββ- // All math in fp32. cos_sin_cache is loaded as fp32 (its native storage).- bool const is_rope_lane = dim_base >= kNopeDim;- if (is_rope_lane) {- int64_t const pos = position_ids[tokenIdx];- constexpr int kHalfRope = kRopeDim / 2;- float const* cos_ptr = cos_sin_cache + pos * kRopeDim;- float const* sin_ptr = cos_ptr + kHalfRope;-- int const rope_local_base = dim_base - kNopeDim;- int const half_base = rope_local_base >> 1;-- // Load phase: 4 vectorized LDGs issue back-to-back.- float4 const c0 = *reinterpret_cast<float4 const*>(cos_ptr + half_base);- float4 const c1 = *reinterpret_cast<float4 const*>(cos_ptr + half_base + 4);- float4 const s0 = *reinterpret_cast<float4 const*>(sin_ptr + half_base);- float4 const s1 = *reinterpret_cast<float4 const*>(sin_ptr + half_base + 4);- float const cos_arr[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w};- float const sin_arr[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w};--#pragma unroll- for (int p = 0; p < kElemsPerLane / 2; p++) {- float const x_even = elements[2 * p];- float const x_odd = elements[2 * p + 1];- elements[2 * p] = x_even * cos_arr[p] - x_odd * sin_arr[p];- elements[2 * p + 1] = x_even * sin_arr[p] + x_odd * cos_arr[p];- }- }-- // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ- // Q / KV branch dispatch. Restructured as if/else (no early `return`)- // so every code path lands at the same exit point β callers own PDL- // triggering and per-iteration buffer rotation.- // βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ- if (!isKV) {- // ββ Live-Q: cast back to bf16 and store into the padded q_out. βββββ- uint4 out0, out1;- typename Converter::packed_hip_type* po0 =- reinterpret_cast<typename Converter::packed_hip_type*>(&out0);- typename Converter::packed_hip_type* po1 =- reinterpret_cast<typename Converter::packed_hip_type*>(&out1);-#pragma unroll- for (int i = 0; i < 4; i++) {- po0[i] =- Converter::convert(make_float2(elements[2 * i], elements[2 * i + 1]));- }-#pragma unroll- for (int i = 0; i < 4; i++) {- po1[i] = Converter::convert(- make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1]));- }- scalar_t_in* dst =- q_out +- (static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *- kHeadDim +- dim_base;- *reinterpret_cast<uint4*>(dst) = out0;- *reinterpret_cast<uint4*>(dst + 8) = out1;- } else {- // ββ KV: FP8 quant on NoPE + bf16 store on RoPE + cache insert.- int64_t const slot_id = slot_mapping[tokenIdx];- if (slot_id >= 0) {- int64_t const block_idx = slot_id / cache_block_size;- int64_t const pos_in_block = slot_id % cache_block_size;- uint8_t* block_base =- k_cache + block_idx * static_cast<int64_t>(kv_block_stride);- uint8_t* token_fp8_ptr = block_base + pos_in_block * kTokenDataBytes;- uint8_t* token_bf16_ptr = token_fp8_ptr + kNopeDim;- uint8_t* token_scale_ptr =- block_base +- static_cast<int64_t>(cache_block_size) * kTokenDataBytes +- pos_in_block * kScaleBytesPerToken;--#pragma unroll- for (int i = 0; i < kElemsPerLane; i++) {- elements[i] = Converter::convert(Converter::convert(elements[i]));- }-- float local_absmax = 0.0f;-#pragma unroll- for (int i = 0; i < kElemsPerLane; i++) {- local_absmax = fmaxf(local_absmax, fabsf(elements[i]));- }- float const absmax = fmaxf(warp4MaxAbs(local_absmax), 1e-4f);- float const exponent = ceilf(log2f(absmax / kFp8Max));- float const inv_scale = exp2f(-exponent);-- if (!is_rope_lane) {- uint8_t out_bytes[kElemsPerLane];-#pragma unroll- for (int i = 0; i < kElemsPerLane; i++) {- float scaled = elements[i] * inv_scale;- scaled = fminf(fmaxf(scaled, -kFp8Max), kFp8Max);-#ifndef USE_ROCM- __nv_fp8_storage_t s =- __nv_cvt_float_to_fp8(scaled, __NV_SATFINITE, __NV_E4M3);- out_bytes[i] = static_cast<uint8_t>(s);-#else- out_bytes[i] = rocm_cvt_float_to_fp8_e4m3(scaled);-#endif- }- *reinterpret_cast<uint4*>(token_fp8_ptr + dim_base) =- *reinterpret_cast<uint4 const*>(out_bytes);-- if ((laneId & 3) == 0) {- int const q_block_idx = laneId >> 2;- float encoded = fmaxf(fminf(exponent + 127.0f, 255.0f), 0.0f);- token_scale_ptr[q_block_idx] = static_cast<uint8_t>(encoded);- }- if (laneId == 0) {- token_scale_ptr[kNumQuantBlocks] = 0;- }- } else {- uint4 out0, out1;- typename Converter::packed_hip_type* po0 =- reinterpret_cast<typename Converter::packed_hip_type*>(&out0);- typename Converter::packed_hip_type* po1 =- reinterpret_cast<typename Converter::packed_hip_type*>(&out1);-#pragma unroll- for (int i = 0; i < 4; i++) {- po0[i] = Converter::convert(- make_float2(elements[2 * i], elements[2 * i + 1]));- }-#pragma unroll- for (int i = 0; i < 4; i++) {- po1[i] = Converter::convert(- make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1]));- }- int const rope_local_base = dim_base - kNopeDim;- scalar_t_in* bf16_dst =- reinterpret_cast<scalar_t_in*>(token_bf16_ptr) + rope_local_base;- *reinterpret_cast<uint4*>(bf16_dst) = out0;- *reinterpret_cast<uint4*>(bf16_dst + 8) = out1;- }- }- }-}--// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-// Kernel-// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-//-// Grid: 1D, gridDim.x = ceil(num_tokens_full * (kNumHeadsQPadded + 1) /-// warps_per_block) Block: blockDim.x = 256 threads (8 warps per block) Each-// warp handles one (token, head_slot) pair.-// slot < num_heads_q β live-Q branch-// (RMSNorm + RoPE,-// read q_in β write q_out)-// num_heads_q <= slot < kNumHeadsQPadded β pad-Q branch-// (zero-fill q_out)-// slot == kNumHeadsQPadded β KV branch-// (RoPE + UE8M0 quant +-// paged-cache insert)-//-// `kNumHeadsQPadded` is a template parameter (compile-time constant) so the-// divisions in the grid math and the KV-sentinel comparison fold to fast-// constant operations. The launch wrapper dispatches the runtime value to-// the matching instantiation.-//-// With DP padding, q/kv/position_ids can have more rows than slot_mapping.-// The live-Q and pad-Q branches cover all `num_tokens_full` rows (downstream-// attention uses them). The KV branch only inserts the first-// `num_tokens_insert` tokens (= slot_mapping length) into the paged cache.-//-template <typename scalar_t_in, int kNumHeadsQPadded>-__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(- scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512]- scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512]- scalar_t_in const* __restrict__ kv_in, // [N, 512] bf16- uint8_t* __restrict__ k_cache, // [num_blocks, block_stride]- int64_t const* __restrict__ slot_mapping, // [num_tokens_insert] i64- int64_t const* __restrict__ position_ids, // [N] i64- float const* __restrict__ cos_sin_cache, // [max_pos, 64] fp32- float const eps,- int const num_tokens_full, // = q.size(0) = kv.size(0)- int const num_tokens_insert, // = slot_mapping.size(0), β€ num_tokens_full- int const num_heads_q, // live Q heads (input layout)- int const cache_block_size, // tokens per paged-cache block- int const kv_block_stride) { // bytes per paged-cache block-#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)- // BF16 _typeConvert specialization is unavailable on pre-Ampere. The- // DeepseekV4 kernel only runs with bf16 inputs in practice, so compile a- // no-op stub for sm_70/sm_75 to keep multi-arch builds happy.- if constexpr (std::is_same_v<scalar_t_in, __nv_bfloat16>) {- return;- } else {-#endif- int const warpsPerBlock = blockDim.x / 32;- int const warpId = threadIdx.x / 32;- int const laneId = threadIdx.x % 32;- int const globalWarpIdx = blockIdx.x * warpsPerBlock + warpId;-- constexpr int kTotalSlotsPerToken = kNumHeadsQPadded + 1;- int const tokenIdx = globalWarpIdx / kTotalSlotsPerToken;- int const slotIdx = globalWarpIdx % kTotalSlotsPerToken;- if (tokenIdx >= num_tokens_full) return;-- bool const isKV = (slotIdx == kNumHeadsQPadded);- bool const isPadQ = !isKV && (slotIdx >= num_heads_q);- // KV branch: skip DP-padded tokens (no slot reserved for them).- if (isKV && tokenIdx >= num_tokens_insert) return;-- // PDL: wait for predecessor kernel (upstream q/kv producer) to signal- // before touching any global memory. No-op when PDL is not enabled on- // the launch. The CUDA runtime wrapper emits the griddepcontrol.wait- // PTX with the required memory clobber internally.-#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)- cudaGridDependencySynchronize();-#endif-- // Dim range this lane owns within the 512-wide head.- int const dim_base = laneId * kElemsPerLane; // in [0, 512) step 16-- // Load only for live-Q and KV slots; pad-Q skips the read (q_in beyond- // num_heads_q is out of bounds) and the helper zero-fills its output.- uint4 v0, v1;- if (!isPadQ) {- scalar_t_in const* src_ptr;- if (isKV) {- src_ptr = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;- } else {- int64_t const q_row_offset =- (static_cast<int64_t>(tokenIdx) * num_heads_q + slotIdx) *- kHeadDim +- dim_base;- src_ptr = q_in + q_row_offset;- }- v0 = *reinterpret_cast<uint4 const*>(src_ptr);- v1 = *reinterpret_cast<uint4 const*>(src_ptr + 8);- }-- processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded>(- v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out,- k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size,- kv_block_stride);--#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)- cudaTriggerProgrammaticLaunchCompletion();-#endif-#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)- }-#endif-}--// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-// Kernel-// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-//-// Grid: 1D, gridDim.x = num_tokens_full-// Block: blockDim.x = 256 threads (8 warps per block) Each-// warp handles one token, iterating over each head.-// Q branch (RMSNorm + RoPE, in place) head_slot == num_heads_q-// KV branch (RoPE + UE8M0 quant + insert)-//-template <typename scalar_t_in, int kNumHeadsQPadded>-__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(- scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out,- scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache,- int64_t const* __restrict__ slot_mapping,- int64_t const* __restrict__ position_ids,- float const* __restrict__ cos_sin_cache, float const eps,- int const num_tokens_full, int const num_tokens_insert,- int const num_heads_q, int const cache_block_size,- int const kv_block_stride) {-#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)- if constexpr (std::is_same_v<scalar_t_in, __nv_bfloat16>) {- return;- } else {-#endif- int const warpsPerBlock = blockDim.x / 32;- int const warpId = threadIdx.x / 32;- int const laneId = threadIdx.x % 32;-- int const tokenIdx = blockIdx.x;- if (tokenIdx >= num_tokens_full) return;--#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)- cudaGridDependencySynchronize();-#endif-- int const dim_base = laneId * kElemsPerLane; // in [0, 512) step 16- // Slot enumeration: live-Q + pad-Q + (KV if this token has a slot).- int const slot_end = (tokenIdx >= num_tokens_insert)- ? kNumHeadsQPadded- : (kNumHeadsQPadded + 1);-- auto load_slot = [&](int s, uint4& va, uint4& vb) {- // pad-Q slots skip the load β q_in beyond num_heads_q is OOB.- if (s >= num_heads_q && s < kNumHeadsQPadded) return;- scalar_t_in const* src;- if (s == kNumHeadsQPadded) {- src = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;- } else {- src = q_in +- (static_cast<int64_t>(tokenIdx) * num_heads_q +- static_cast<int64_t>(s)) *- kHeadDim +- dim_base;- }- va = *reinterpret_cast<uint4 const*>(src);- vb = *reinterpret_cast<uint4 const*>(src + 8);- };-- if (warpId < slot_end) {- int curr_slot = warpId;- uint4 v0_curr, v1_curr;- load_slot(curr_slot, v0_curr, v1_curr);-- while (curr_slot < slot_end) {- int const next_slot = curr_slot + warpsPerBlock;- bool const has_next = (next_slot < slot_end);-- // Prefetch src for the next slot- uint4 v0_next, v1_next;- if (has_next) {- load_slot(next_slot, v0_next, v1_next);- }-- processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded>(- v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId,- num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids,- cos_sin_cache, cache_block_size, kv_block_stride);-- // ββ Buffer rotation: hand the prefetched LDGs to the next iter.- v0_curr = v0_next;- v1_curr = v1_next;- curr_slot = next_slot;- } // while- } // if (warpId < slot_end)--#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)- cudaTriggerProgrammaticLaunchCompletion();-#endif-#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)- }-#endif-}--// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-// Launch wrapper-// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-template <typename scalar_t_in, int kNumHeadsQPadded>-static void launchFusedDeepseekV4Templated(- scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,- uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids,- float const* cos_sin_cache, float const eps, int const num_tokens_full,- int const num_tokens_insert, int const num_heads_q,- int const cache_block_size, int const kv_block_stride,- cudaStream_t stream) {- constexpr int kBlockSize = 256;- constexpr int kWarpsPerBlock = kBlockSize / 32;- int64_t const total_warps =- static_cast<int64_t>(num_tokens_full) * (kNumHeadsQPadded + 1);- int const grid =- static_cast<int>((total_warps + kWarpsPerBlock - 1) / kWarpsPerBlock);-- // PDL: enable programmatic stream serialization whenever the hardware- // supports it (SM90+). On pre-Hopper GPUs the attribute is unavailable,- // so leave numAttrs = 0 and launch as a regular kernel.-#ifndef USE_ROCM- static int const sm_version = getSMVersion();- // Host-side guard: the device kernel body is compiled as a no-op for- // bf16 on pre-Ampere (sm_70/sm_75) because _typeConvert<BFloat16> is- // unavailable there. Refuse the launch loudly instead of silently- // skipping the work.- if (sm_version < 80) return;- cudaLaunchConfig_t config;- config.gridDim = dim3(grid);- config.blockDim = dim3(kBlockSize);- config.dynamicSmemBytes = 0;- config.stream = stream;- cudaLaunchAttribute attrs[1];- attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;- attrs[0].val.programmaticStreamSerializationAllowed = 1;- config.attrs = attrs;- config.numAttrs = (sm_version >= 90) ? 1 : 0;-- if (num_tokens_full < NUM_TOKEN_CUTOFF) {- cudaLaunchKernelEx(- &config,- fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<scalar_t_in,- kNumHeadsQPadded>,- q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,- eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,- kv_block_stride);- } else {- config.gridDim = dim3(num_tokens_full);- cudaLaunchKernelEx(- &config,- fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid<- scalar_t_in, kNumHeadsQPadded>,- q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,- eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,- kv_block_stride);- }-#else- // ROCm: use standard kernel launch syntax (no PDL/stream serialization)- // clang-format off- fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<scalar_t_in, kNumHeadsQPadded>- <<<grid, kBlockSize, 0, stream>>>(- q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,- cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q,- cache_block_size, kv_block_stride);-#endif-}--// Runtime dispatch into one of the precompiled `kNumHeadsQPadded`-// instantiations. Supported padded head counts: 8, 16, 32, 64, 128.-template <typename scalar_t_in>-void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(- scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,- uint8_t* k_cache, int64_t const* slot_mapping,- int64_t const* position_ids, float const* cos_sin_cache, float const eps,- int const num_tokens_full, int const num_tokens_insert,- int const num_heads_q, int const num_heads_q_padded,- int const cache_block_size, int const kv_block_stride,- cudaStream_t stream) {-#define DISPATCH(N) \- case N: \- launchFusedDeepseekV4Templated<scalar_t_in, N>( \- q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \- cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q, \- cache_block_size, kv_block_stride, stream); \- return;-- switch (num_heads_q_padded) {- DISPATCH(8)- DISPATCH(16)- DISPATCH(32)- DISPATCH(64)- DISPATCH(128)- default: return;- }-#undef DISPATCH-}--// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ-// FlashInfer full-cache kernel-// ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ--} // namespace deepseek_v4_fused_ops-} // namespace vllm--extern "C" int dsv4_vllm_qnorm_rope_kv_insert_batch(- const __nv_bfloat16 *q_in, __nv_bfloat16 *q_out,- const __nv_bfloat16 *kv_in, uint8_t *kv_cache,- const int64_t *slot_mapping, const int64_t *position_ids,- const float *cos_sin_cache, float eps, int tokens, int insert_tokens,- int heads, int padded_heads,- int cache_block_size, int kv_block_stride, cudaStream_t stream) {- if (!q_in || !q_out || !kv_in || !kv_cache || !slot_mapping ||- !position_ids || !cos_sin_cache || tokens < 1 || insert_tokens < 0 ||- insert_tokens > tokens) return 0;- vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(- q_in, q_out, kv_in, kv_cache, slot_mapping, position_ids,- cos_sin_cache, eps, tokens, insert_tokens, heads, padded_heads, cache_block_size,- kv_block_stride, stream);- return cudaGetLastError() == cudaSuccess;-}--extern "C" int dsv4_vllm_qnorm_rope_kv_insert(- const __nv_bfloat16 *q_in, __nv_bfloat16 *q_out,- const __nv_bfloat16 *kv_in, uint8_t *kv_cache,- const int64_t *slot_mapping, const int64_t *position_ids,- const float *cos_sin_cache, float eps, int heads,- int cache_block_size, int kv_block_stride, cudaStream_t stream) {- return dsv4_vllm_qnorm_rope_kv_insert_batch(- q_in, q_out, kv_in, kv_cache, slot_mapping, position_ids, cos_sin_cache,- eps, 1, 1, heads, heads, cache_block_size, kv_block_stride, stream);-}--__global__ void packCompressedKV(const float *kv,uint8_t *cache,const int *state,int ratio,- int cache_block_size,int kv_block_stride) {- int pos=state[1];if(!ratio||(pos+1)%ratio)return;int slot=(pos+1)/ratio-1;kv+=(long long)slot*512;- int tile=blockIdx.x,lane=threadIdx.x,block=slot/cache_block_size,row=slot%cache_block_size;- uint8_t *base=cache+(long long)block*kv_block_stride;- if(tile<7){- __shared__ float mx[64];float v=__bfloat162float(__float2bfloat16(kv[tile*64+lane]));- mx[lane]=fabsf(v);__syncthreads();- for(int n=32;n;n>>=1){if(lane<n)mx[lane]=fmaxf(mx[lane],mx[lane+n]);__syncthreads();}- float exponent=ceilf(log2f(fmaxf(mx[0],1e-4f)/448.f)),scaled=fminf(fmaxf(v*exp2f(-exponent),-448.f),448.f);- base[(long long)row*576+tile*64+lane]=(uint8_t)__nv_cvt_float_to_fp8(scaled,__NV_SATFINITE,__NV_E4M3);- if(!lane)base[(long long)cache_block_size*576+(long long)row*8+tile]=(uint8_t)fminf(fmaxf(exponent+127.f,0.f),255.f);- }else{- __nv_bfloat16 *rope=(__nv_bfloat16*)(base+(long long)row*576+448);- rope[lane]=__float2bfloat16(kv[448+lane]);- if(!lane)base[(long long)cache_block_size*576+(long long)row*8+7]=0;- }-}--extern "C" int dsv4_vllm_pack_compressed(const float *kv,uint8_t *cache,const int *decode_state,- int ratio,- int cache_block_size,int kv_block_stride,- cudaStream_t stream){- if(!kv||!cache||!decode_state||ratio<1||cache_block_size<1)return 0;- packCompressedKV<<<8,64,0,stream>>>(kv,cache,decode_state,ratio,cache_block_size,kv_block_stride);- return cudaGetLastError()==cudaSuccess;-}diff --git a/c/backend_cuda_dsv4_qkv_vllm.h b/c/backend_cuda_dsv4_qkv_vllm.hdeleted file mode 100644index dd5d7281..00000000--- a/c/backend_cuda_dsv4_qkv_vllm.h+++ /dev/null@@ -1,26 +0,0 @@-#ifndef COLIBRI_BACKEND_CUDA_DSV4_QKV_VLLM_H-#define COLIBRI_BACKEND_CUDA_DSV4_QKV_VLLM_H-#include <cuda_runtime.h>-#include <stdint.h>-#ifdef __cplusplus-extern "C" {-#endif-int dsv4_vllm_qnorm_rope_kv_insert(const void *q_in,void *q_out,const void *kv_in,- uint8_t *kv_cache,const int64_t *slot_mapping,- const int64_t *position_ids,const float *cos_sin_cache,- float eps,int heads,int cache_block_size,- int kv_block_stride,cudaStream_t stream);-int dsv4_vllm_qnorm_rope_kv_insert_batch(const void *q_in,void *q_out,const void *kv_in,- uint8_t *kv_cache,const int64_t *slot_mapping,- const int64_t *position_ids,const float *cos_sin_cache,- float eps,int tokens,int insert_tokens,int heads,- int padded_heads,int cache_block_size,- int kv_block_stride,cudaStream_t stream);-int dsv4_vllm_pack_compressed(const float *kv,uint8_t *cache,const int *decode_state,- int ratio,- int cache_block_size,int kv_block_stride,- cudaStream_t stream);-#ifdef __cplusplus-}-#endif-#endifdiff --git a/c/tests/bench_dsv4_flashinfer.cu b/c/tests/bench_dsv4_flashinfer.cudeleted file mode 100644index 4b9ff790..00000000--- a/c/tests/bench_dsv4_flashinfer.cu+++ /dev/null@@ -1,81 +0,0 @@-#include <cuda_runtime.h>-#include <flashinfer/gemm/group_gemm_mxfp4_groupwise_sm120.cuh>-#include <cmath>-#include <cstdlib>-#include <cstdio>-#include <vector>--namespace flashinfer::group_gemm {-INSTANTIATE_GROUP_GEMM_MXFP4_GROUPWISE_SM120(- 128, 32, 128, true, cutlass::float_e4m3_t, cutlass::float_e2m1_t,- cutlass::float_ue8m0_t, cutlass::float_ue8m0_t, cutlass::bfloat16_t,- fp8, fp4, ue8m0, ue8m0, bf16)-}--using namespace flashinfer::group_gemm;--static void check(cudaError_t e, const char *where) {- if (e != cudaSuccess) {- std::fprintf(stderr, "%s: %s\n", where, cudaGetErrorString(e));- std::exit(1);- }-}--int main(int argc, char **argv) {- int groups = argc > 1 ? std::atoi(argv[1]) : 6;- int n = argc > 2 ? std::atoi(argv[2]) : 2048;- int k = argc > 3 ? std::atoi(argv[3]) : 4096;- constexpr int rows = 4;- int total_rows = groups * rows;- constexpr size_t int_workspace_bytes = 8 << 20;- constexpr size_t float_workspace_bytes = 64 << 20;- cutlass::float_e4m3_t *a;- cutlass::float_e2m1_t *b;- cutlass::float_ue8m0_t *as, *bs;- cutlass::bfloat16_t *out;- int *indptr;- void *int_workspace, *float_workspace;- check(cudaMalloc(&a, (size_t)total_rows * k), "allocate activation");- check(cudaMalloc(&b, (size_t)groups * n * k / 2), "allocate weight");- check(cudaMalloc(&as, (size_t)groups * 128 * (k / 32)), "allocate activation scales");- check(cudaMalloc(&bs, (size_t)groups * n * (k / 32)), "allocate weight scales");- check(cudaMalloc(&out, (size_t)total_rows * n * sizeof(*out)), "allocate output");- check(cudaMalloc(&indptr, (groups + 1) * sizeof(*indptr)), "allocate indptr");- check(cudaMalloc(&int_workspace, int_workspace_bytes), "allocate integer workspace");- check(cudaMalloc(&float_workspace, float_workspace_bytes), "allocate CUTLASS workspace");- check(cudaMemset(a, 0x38, (size_t)total_rows * k), "initialize unit FP8 activation");- check(cudaMemset(b, 0x22, (size_t)groups * n * k / 2), "initialize unit FP4 weight");- check(cudaMemset(as, 127, (size_t)groups * 128 * (k / 32)), "initialize activation scales");- check(cudaMemset(bs, 127, (size_t)groups * n * (k / 32)), "initialize weight scales");- std::vector<int> host_indptr(groups + 1);- for (int i = 0; i <= groups; ++i) host_indptr[i] = i * rows;- check(cudaMemcpy(indptr, host_indptr.data(), (groups + 1) * sizeof(*indptr), cudaMemcpyHostToDevice),- "upload indptr");- auto run = [&] {- check(CutlassMXFP4GroupwiseScaledGroupGEMMSM120<- 128, 32, 128, true, cutlass::float_e4m3_t, cutlass::float_e2m1_t,- cutlass::float_ue8m0_t, cutlass::float_ue8m0_t, cutlass::bfloat16_t>(- int_workspace, int_workspace_bytes, float_workspace, float_workspace_bytes,- a, b, as, bs, out, indptr, n, k, groups, nullptr, 0),- "launch grouped MXFP4 GEMM");- };- for (int i = 0; i < 10; ++i) run();- check(cudaDeviceSynchronize(), "warmup");- cutlass::bfloat16_t first;- check(cudaMemcpy(&first, out, sizeof(first), cudaMemcpyDeviceToHost), "download correctness sample");- if (std::abs(float(first) - k) > k / 100.f) {- std::fprintf(stderr, "wrong MXFP4 result: got %.3f expected %d\n", float(first), k);- return 2;- }- cudaEvent_t begin, end;- check(cudaEventCreate(&begin), "create begin event");- check(cudaEventCreate(&end), "create end event");- check(cudaEventRecord(begin), "record begin");- for (int i = 0; i < 100; ++i) run();- check(cudaEventRecord(end), "record end");- check(cudaEventSynchronize(end), "benchmark");- float elapsed_ms = 0;- check(cudaEventElapsedTime(&elapsed_ms, begin, end), "elapsed time");- std::printf("flashinfer_sm120_mxfp4 groups=%d n=%d k=%d %.3f us/call\n",- groups, n, k, elapsed_ms * 10.f);-}