Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Remove vendored DeepSeek V4 CUDA paths

ZacharyZcR763dac7882da
diff --git a/c/Makefile b/c/Makefile
index 747f63b9..d2eb7423 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -175,7 +175,6 @@ CUDA_DLL ?= 0
COLI_ANS ?= 0
DIETGPU_ROOT ?=
DEEPGEMM ?= 0
-VLLM_MHC ?= 0
NCCL ?= 0
ifneq ($(IS_WIN),)
# the CUDA installer sets CUDA_PATH system-wide (e.g. C:\Program Files\NVIDIA
@@ -437,23 +436,10 @@ endif
# objects are self-contained: backend_cuda_dsv4.* include only their own header
# and the CUDA runtime, so they build and their tests run without the engine.
#
-# VLLM_MHC, DEEPGEMM and FLASHINFER below are UNSUPPORTED. They are eight build
-# configurations and CI compiles none of them: each needs an external checkout
-# (TILELANG_HOME, DEEPGEMM_HOME, FLASHINFER_HOME) and DeepGEMM's kernels want
-# Blackwell to run. That is a statement about verification, not quality -- an
-# untested build path rots silently, and whoever is here in six months inherits
-# it. They gate nothing and block nothing: the default build (all three off) is
-# the one CI compiles and the one to report bugs against. Enable them only if
-# you have the external trees and are prepared to debug the combination.
+# DEEPGEMM is unsupported: CI cannot compile it without an external checkout,
+# and its kernels require Blackwell to run. It gates nothing and blocks nothing;
+# the default build is the supported configuration.
DSV4_CUDA_OBJ = backend_cuda_dsv4.o
-ifeq ($(VLLM_MHC),1)
-ifndef TILELANG_HOME
-$(error TILELANG_HOME must point to an installed TileLang package)
-endif
-CFLAGS += -DCOLI_DSV4_VLLM_MHC
-override NVCCFLAGS += -DCOLI_DSV4_VLLM_MHC
-DSV4_CUDA_OBJ += backend_cuda_dsv4_mhc_vllm.o
-endif
ifeq ($(DEEPGEMM),1)
ifndef DEEPGEMM_HOME
$(error DEEPGEMM_HOME must point to a DeepGEMM include tree)
@@ -469,14 +455,6 @@ override NVCC_STD := c++20
override NVCCFLAGS += --expt-relaxed-constexpr --expt-extended-lambda -DCOLI_DSV4_DEEPGEMM -I"$(DEEPGEMM_HOME)"
LDFLAGS += -lcuda
endif
-ifeq ($(FLASHINFER),1)
-ifndef FLASHINFER_HOME
-$(error FLASHINFER_HOME must point to a FlashInfer source checkout)
-endif
-CFLAGS += -DCOLI_DSV4_FLASHINFER
-override NVCCFLAGS += -DCOLI_DSV4_FLASHINFER
-DSV4_CUDA_OBJ += backend_cuda_dsv4_flashinfer.o backend_cuda_dsv4_sparse_mla_flashinfer.o backend_cuda_dsv4_qkv_vllm.o
-endif
ifeq ($(NCCL),1)
CFLAGS += -DCOLI_DSV4_NCCL
override NVCCFLAGS += -DCOLI_DSV4_NCCL
@@ -687,32 +665,9 @@ backend_cuda.o: backend_cuda.cu backend_cuda.h backend_gpu_compat.h .build-confi
"$(GPUCC)" $(GPUFLAGS) -c backend_cuda.cu -o $@
-backend_cuda_dsv4.o: backend_cuda_dsv4.cu backend_cuda_dsv4.h backend_cuda_dsv4_flashinfer.h .build-config
+backend_cuda_dsv4.o: backend_cuda_dsv4.cu backend_cuda_dsv4.h .build-config
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
"$(NVCC)" $(NVCCFLAGS) -c backend_cuda_dsv4.cu -o $@
-
-backend_cuda_dsv4_mhc_vllm.o: backend_cuda_dsv4_mhc_vllm.cu backend_cuda_dsv4_mhc_vllm.h .build-config
- @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
- "$(NVCC)" -O3 -std=c++20 -arch=sm_120 -DENABLE_BF16 \
- -I"$(TILELANG_HOME)/src" -I"$(TILELANG_HOME)/3rdparty/cutlass/include" \
- -c backend_cuda_dsv4_mhc_vllm.cu -o $@
-
-backend_cuda_dsv4_flashinfer.o: backend_cuda_dsv4_flashinfer.cu backend_cuda_dsv4_flashinfer.h .build-config
- @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
- "$(NVCC)" -O3 -std=c++17 -arch=sm_120a --expt-relaxed-constexpr -diag-suppress=20012 -diag-suppress=20013 -diag-suppress=20015 \
- -I"$(FLASHINFER_HOME)/include" -I"$(FLASHINFER_HOME)/3rdparty/cutlass/include" \
- -I"$(FLASHINFER_HOME)/3rdparty/cutlass/tools/util/include" -c backend_cuda_dsv4_flashinfer.cu -o $@
-
-backend_cuda_dsv4_sparse_mla_flashinfer.o: $(FLASHINFER_HOME)/csrc/sparse_mla_sm120_decode_dsv4.cu .build-config
- @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
- "$(NVCC)" -O3 -std=c++17 -gencode=arch=compute_120f,code=sm_120f --expt-relaxed-constexpr \
- -I"$(FLASHINFER_HOME)/include" -I"$(FLASHINFER_HOME)/3rdparty/cutlass/include" \
- -c "$<" -o $@
-
-backend_cuda_dsv4_qkv_vllm.o: backend_cuda_dsv4_qkv_vllm.cu backend_cuda_dsv4_qkv_vllm.h .build-config
- @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
- "$(NVCC)" -O3 -std=c++17 -gencode=arch=compute_120f,code=sm_120f \
- --expt-relaxed-constexpr --expt-extended-lambda -c "$<" -o $@
backend_metal.o: backend_metal.mm backend_metal.h
$(METALXX) -c backend_metal.mm -o $@
diff --git a/c/THIRD_PARTY_NOTICES.md b/c/THIRD_PARTY_NOTICES.md
deleted file mode 100644
index dde8a3c6..00000000
--- a/c/THIRD_PARTY_NOTICES.md
+++ /dev/null
@@ -1,60 +0,0 @@
-# Third-party notices
-
-Portions of `backend_cuda_dsv4.cu`, including the DeepSeek-V4 GPU router
-selection algorithm, are adapted from `ds4_cuda.cu` in the ds4 project:
-https://github.com/antirez/ds4
-
-MIT License
-
-Copyright (c) 2026 The ds4.c authors
-Copyright (c) 2023-2026 The ggml authors
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-`tests/bench_dsv4_flashinfer.cu` instantiates the FlashInfer SM120 grouped
-MXFP4 GEMM interface from https://github.com/flashinfer-ai/flashinfer.
-
-Copyright (c) 2023-2026 FlashInfer authors
-
-Licensed under the Apache License, Version 2.0. You may obtain a copy at
-https://www.apache.org/licenses/LICENSE-2.0.
-
-The DeepSeek-V4 DeepGEMM oracle and benchmark tools follow the FP8 packing
-and kernel-dispatch interfaces used by the vLLM project:
-https://github.com/vllm-project/vllm
-
-The pinned DeepSeek-V4 reference runner in `tools/dsv4_vllm_reference.py`
-executes vLLM commit `ffd46bfab2128bb84146050e98b51a617c6575ab`
-without replacing or modifying its kernels. It is retained as the behavioral
-and performance oracle for the native Colibri port.
-
-`backend_cuda_dsv4_mhc_vllm.cu` is the CUDA source generated from that exact
-commit's TileLang mHC decode kernels for the DeepSeek-V4-Flash configuration
-(`hc=4`, `hidden=4096`, `n_out=24`, `split_k=8`). It is kept as the pinned
-upstream source for the native ABI integration.
-
-`backend_cuda_dsv4_qkv_vllm.cu` adapts that commit's
-`fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu`. The Torch registration
-layer is replaced by a native C ABI; the CUDA Q RMSNorm, RoPE, UE8M0 FP8
-quantization, and paged KV-cache insertion algorithm remains upstream code.
-
-Copyright contributors to the vLLM project
-
-Licensed under the Apache License, Version 2.0. You may obtain a copy at
-https://www.apache.org/licenses/LICENSE-2.0.
diff --git a/c/backend_cuda_dsv4.cu b/c/backend_cuda_dsv4.cu
index ce924944..d5bfc52b 100644
--- a/c/backend_cuda_dsv4.cu
+++ b/c/backend_cuda_dsv4.cu
@@ -11,34 +11,24 @@
#ifdef COLI_DSV4_NCCL
#include <nccl.h>
#endif
-#ifdef COLI_DSV4_VLLM_MHC
-#include "backend_cuda_dsv4_mhc_vllm.h"
-#endif
#ifdef COLI_DSV4_DEEPGEMM
#include <cuda.h>
#include <deep_gemm/impls/sm120_fp8_fp4_gemm_1d1d.cuh>
#include <deep_gemm/impls/sm120_split_k_reduce.cuh>
#include <deep_gemm/impls/sm120_bf16_gemm.cuh>
#endif
-#ifdef COLI_DSV4_FLASHINFER
-#include "backend_cuda_dsv4_flashinfer.h"
-#include "backend_cuda_dsv4_qkv_vllm.h"
-#endif
-struct Dsv4CudaTensor { void *w,*bf16_cache; uint8_t *scale,*fi_scale;int *dg_scale; int O,I,fmt,device,own_w,own_scale,own_dg_scale; long long bytes; };
+struct Dsv4CudaTensor { void *w,*bf16_cache; uint8_t *scale;int *dg_scale; int O,I,fmt,device,own_w,own_scale,own_dg_scale; long long bytes; };
struct Dsv4CudaActivation { float *data; long long elements; int device; };
struct Dsv4CudaGraph { cudaGraph_t graph;cudaGraphExec_t exec;int device; };
-struct Dsv4CudaKvCache { float *kv,*compressed,*comp_value,*comp_score,*rope_cos,*rope_sin,*comp_cos,*comp_sin,*fi_lse,*fi_out_lse,*fi_cos_sin;uint8_t *fi_kv,*fi_comp;__nv_bfloat16 *fi_q,*fi_mid,*fi_out;int *compressed_len,*fi_indices,*fi_comp_indices,*fi_length,*fi_comp_length;int64_t *fi_slot,*fi_pos;size_t fi_qcap,fi_midcap,fi_outcap,fi_lsecap,fi_out_lsecap,fi_indicescap,fi_lengthcap,fi_slotcap,fi_poscap;int device,window,dim,max_tokens,max_compressed,rope_pairs,fi_splits; };
+struct Dsv4CudaKvCache { float *kv,*compressed,*comp_value,*comp_score,*rope_cos,*rope_sin,*comp_cos,*comp_sin;int *compressed_len;int device,window,dim,max_tokens,max_compressed,rope_pairs; };
struct MvDesc { const uint8_t *w,*scale; const float *x; float *y; };
-struct ExpertPtr { const uint8_t *w,*scale,*fi_scale; };
+struct ExpertPtr { const uint8_t *w,*scale; };
struct Dsv4CudaExpertSet { ExpertPtr *table;int64_t *hash;Dsv4CudaTensor *sg,*su,*sd;uint8_t *bank_fc1,*bank_fc1_scale,*bank_fc2,*bank_fc2_scale;int *bank_fc1_dg_scale,*bank_fc2_dg_scale;int count,device,H,I,vocab,topk; };
struct TcPlan {int O,I,ok;cublasLtMatmulDesc_t op;cublasLtMatrixLayout_t a,b,c,d;cublasLtMatmulHeuristicResult_t algo;};
struct Dev { int id; cudaStream_t stream,aux; cudaEvent_t ready,fork,join;float *dx,*dy,*p1,*p2,*p3,*p4,*aux1,*aux2,*aux3; size_t xcap,ycap,p1cap,p2cap,p3cap,p4cap,aux1cap,aux2cap,aux3cap; float *hx,*hy;
#ifdef COLI_DSV4_NCCL
ncclComm_t ep_comm;
-#endif
-#ifdef COLI_DSV4_VLLM_MHC
- __nv_bfloat16 *mhcb1,*mhcb2,*mhcb3;size_t mhcb1cap,mhcb2cap,mhcb3cap;
#endif
int *decode_state;
MvDesc *mvdesc;size_t mvdesccap;float *expert_weights;size_t expert_weightscap;int *expert_ids;size_t expert_idcap;
@@ -56,17 +46,8 @@ static int nok(ncclResult_t e,const char*s){if(e==ncclSuccess)return 1;fprintf(s
#endif
static int buf(void **p,size_t *cap,size_t n){if(*cap>=n)return 1;if(*p)cudaFree(*p);*p=nullptr;*cap=0;if(!ok(cudaMalloc(p,n),"buffer allocation"))return 0;*cap=n;return 1;}
-#ifdef COLI_DSV4_VLLM_MHC
-__global__ void mhc_float_to_bf16(__nv_bfloat16*out,const float*in,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=__float2bfloat16(in[i]);}
-__global__ void mhc_bf16_to_float(float*out,const __nv_bfloat16*in,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)out[i]=__bfloat162float(in[i]);}
-#endif
-
__constant__ float e8_table[256];
__global__ void set_decode_state(int*state,int token,int position){if(!threadIdx.x){state[0]=token;state[1]=position;}}
-#ifdef COLI_DSV4_FLASHINFER
-__device__ __forceinline__ int fi_sf_off(int row,int col,int cols){return (col&3)+(col/4)*512+(row&31)*16+((row&127)/32)*4+(row/128)*128*((cols+3)&~3);}
-__global__ void fi_swizzle_scale(uint8_t*out,const uint8_t*in,int O,int cols){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)O*cols;if(p<n){int row=p/cols,col=p%cols;out[fi_sf_off(row,col,cols)]=in[p];}}
-#endif
__device__ float e4m3(uint8_t b){int sign=b>>7,e=(b>>3)&15,m=b&7;float v;
if(!e)v=ldexpf((float)m,-9);else if(e==15)v=m==7?NAN:ldexpf(1.f+m/8.f,8);else v=ldexpf(1.f+m/8.f,e-7);
return sign?-v:v;}
@@ -579,24 +560,8 @@ __device__ __forceinline__ int decode_pos(const int *state){return state[1];}
__device__ __forceinline__ int compression_ready(const int *state,int ratio){return ratio&&decode_pos(state)%ratio==ratio-1;}
__global__ void rope_heads_dynamic(float*x,int heads,int dim,int rope,const int*state,int delta,int inverse,int predicate,const float*cs,const float*sn){if(predicate&&!compression_ready(state,predicate))return;int pos=decode_pos(state)+delta,p=blockIdx.x*blockDim.x+threadIdx.x,pairs=rope/2;if(p<heads*pairs){int h=p/pairs,k=p%pairs,off=h*dim+dim-rope+2*k;float s=inverse?-sn[(long long)pos*pairs+k]:sn[(long long)pos*pairs+k],u=x[off],v=x[off+1];x[off]=__bfloat162float(__float2bfloat16(u*cs[(long long)pos*pairs+k]-v*s));x[off+1]=__bfloat162float(__float2bfloat16(u*s+v*cs[(long long)pos*pairs+k]));}}
__global__ void cache_store_dynamic(float*cache,const float*kv,const int*state,int window,int dim){int d=blockIdx.x*blockDim.x+threadIdx.x;if(d<dim)cache[(long long)(decode_pos(state)%window)*dim+d]=kv[d];}
-#ifdef COLI_DSV4_FLASHINFER
-__global__ void fi_prepare_q(__nv_bfloat16*q,const float*x,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n)q[i]=__float2bfloat16(x[i]);}
-__global__ void rmsnorm_rows(float*out,const float*in,const float*w,int rows,int H,float eps){int row=blockIdx.x;const float*x=in+(long long)row*H;float*y=out+(long long)row*H;__shared__ float ws[8],inv;int lane=threadIdx.x&31,warp=threadIdx.x>>5;float ss=0;for(int h=threadIdx.x;h<H;h+=blockDim.x){float v=x[h];ss+=v*v;}for(int d=16;d;d>>=1)ss+=__shfl_down_sync(0xffffffff,ss,d);if(!lane)ws[warp]=ss;__syncthreads();if(!warp){float v=lane<8?ws[lane]:0;for(int d=16;d;d>>=1)v+=__shfl_down_sync(0xffffffff,v,d);if(!lane)inv=rsqrtf(v/H+eps);}__syncthreads();for(int h=threadIdx.x;h<H;h+=blockDim.x)y[h]=__bfloat162float(__float2bfloat16(x[h]*inv*w[h]));(void)rows;}
-__global__ void rmsnorm_rows_bf16(float*out,const float*in,const __nv_bfloat16*w,int rows,int H,float eps){int row=blockIdx.x;const float*x=in+(long long)row*H;float*y=out+(long long)row*H;__shared__ float ws[8],inv;int lane=threadIdx.x&31,warp=threadIdx.x>>5;float ss=0;for(int h=threadIdx.x;h<H;h+=blockDim.x){float v=x[h];ss+=v*v;}for(int d=16;d;d>>=1)ss+=__shfl_down_sync(0xffffffff,ss,d);if(!lane)ws[warp]=ss;__syncthreads();if(!warp){float v=lane<8?ws[lane]:0;for(int d=16;d;d>>=1)v+=__shfl_down_sync(0xffffffff,v,d);if(!lane)inv=rsqrtf(v/H+eps);}__syncthreads();for(int h=threadIdx.x;h<H;h+=blockDim.x)y[h]=__bfloat162float(__float2bfloat16(x[h]*inv*__bfloat162float(w[h])));(void)rows;}
-__global__ void split_qkv_rows(float*q,float*kv,const float*in,int rows,int R,int K){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)rows*(R+K);if(p<n){int row=p/(R+K),col=p%(R+K);if(col<R)q[(long long)row*R+col]=in[p];else kv[(long long)row*K+col-R]=in[p];}}
-__global__ void fi_position_batch(int64_t*slot,int64_t*position,int start,int tokens){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<tokens)slot[i]=position[i]=start+i;}
-__global__ void fi_indices_batch(int*idx,int*length,int start,int tokens,int window,int topk){int t=blockIdx.x,p=threadIdx.x;if(t>=tokens)return;int pos=start+t,n=min(pos+1,window),first=pos+1-n;if(p<topk)idx[(long long)t*topk+p]=p<n?first+p:-1;if(!p)length[t]=n;}
__global__ void gather_group_rows(float*out,const float*in,int tokens,int groups,int width,int group){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)tokens*width;if(p<n){int row=p/width,col=p%width;out[p]=in[((long long)row*groups+group)*width+col];}}
__global__ void scatter_group_rows(float*out,const float*in,int tokens,int groups,int width,int group){long long p=(long long)blockIdx.x*blockDim.x+threadIdx.x,n=(long long)tokens*width;if(p<n){int row=p/width,col=p%width;out[((long long)row*groups+group)*width+col]=in[p];}}
-__global__ void fi_indices(int*idx,int*length,const int*state,int window){int i=threadIdx.x,pos=decode_pos(state),n=min(pos+1,window),start=pos+1-n;if(i<512)idx[i]=i<n?start+i:-1;if(!i)*length=n;}
-/* C4A attends to the indexer's compressed candidates in addition to SWA.
- * The ranges intentionally overlap: those summaries are separate learned KV
- * entries, not duplicates for the runtime to remove. Before the Lightning
- * Indexer is needed, every live compressed entry is therefore a candidate. */
-__global__ void fi_extra_indices(int*idx,int*out_length,const int*length,const int*state,int window,int ratio){int i=threadIdx.x,n=min(*length,512);if(i<512)idx[i]=i<n?i:-1;if(!i)*out_length=n;(void)state;(void)window;(void)ratio;}
-__global__ void fi_position(int64_t*slot,int64_t*position,const int*state){if(!threadIdx.x){*slot=decode_pos(state);*position=decode_pos(state);}}
-__global__ void fi_cos_sin(float*out,const float*cs,const float*sn,int n){int i=blockIdx.x*blockDim.x+threadIdx.x;if(i<n){int pos=i/32,k=i%32;out[(long long)pos*64+k]=cs[i];out[(long long)pos*64+32+k]=sn[i];}}
-#endif
__global__ void compress_store_dynamic(float*values,float*scores,const float*v,const float*g,const float*ape,const int*state,int ratio,int overlap,int O){int d=blockIdx.x*blockDim.x+threadIdx.x;if(d<O){int phase=decode_pos(state)%ratio,slot=(overlap?ratio:0)+phase;values[(long long)slot*O+d]=v[d];scores[(long long)slot*O+d]=g[d]+ape[(long long)phase*O+d];}}
__global__ void compress_pool_dynamic(float*out,const float*values,const float*scores,const int*state,int ratio,int overlap,int dim){if(!compression_ready(state,ratio))return;int d=blockIdx.x*blockDim.x+threadIdx.x;if(d<dim){int O=(1+overlap)*dim,items=overlap?2*ratio:ratio;float mx=-INFINITY;for(int j=0;j<items;j++){int col=overlap&&j>=ratio?dim+d:d;mx=fmaxf(mx,scores[(long long)j*O+col]);}float den=0,sum=0;for(int j=0;j<items;j++){int col=overlap&&j>=ratio?dim+d:d,aoff=j*O+col;float a=expf(scores[aoff]-mx);den+=a;sum+=a*values[aoff];}out[d]=sum/den;}}
__global__ void rmsnorm_dynamic(float*out,const float*x,const float*w,int n,float eps,const int*state,int ratio){if(!compression_ready(state,ratio))return;__shared__ float sum[256],inv;float ss=0;for(int i=threadIdx.x;i<n;i+=blockDim.x)ss+=x[i]*x[i];sum[threadIdx.x]=ss;__syncthreads();for(int k=128;k;k>>=1){if(threadIdx.x<k)sum[threadIdx.x]+=sum[threadIdx.x+k];__syncthreads();}if(!threadIdx.x)inv=rsqrtf(sum[0]/n+eps);__syncthreads();for(int i=threadIdx.x;i<n;i+=blockDim.x)out[i]=__bfloat162float(__float2bfloat16(x[i]*inv*w[i]));}
@@ -638,13 +603,8 @@ __global__ void build_moe_desc(MvDesc *desc,const ExpertPtr *table,const int *id
float *gate,float *up,float *act,float *down,int count,int experts,int H,int I){
int k=threadIdx.x;if(k>=count)return;int e=ids[k];if(e<0||e>=experts)return;
ExpertPtr g=table[e],u=table[experts+e],d=table[2*experts+e];
-#ifdef COLI_DSV4_FLASHINFER
- desc[2*k]={g.w,g.fi_scale,x,gate+(long long)k*I};desc[2*k+1]={u.w,u.fi_scale,x,up+(long long)k*I};
- desc[2*count+k]={d.w,d.fi_scale,act+(long long)k*I,down+(long long)k*H};
-#else
desc[2*k]={g.w,g.scale,x,gate+(long long)k*I};desc[2*k+1]={u.w,u.scale,x,up+(long long)k*I};
desc[2*count+k]={d.w,d.scale,act+(long long)k*I,down+(long long)k*H};
-#endif
}
__global__ void load_hash_ids(int *ids,const int64_t *map,const int *state,int topk){int k=threadIdx.x;if(k<topk)ids[k]=(int)map[(long long)state[0]*topk+k];}
__global__ void mhc_mix_scale(float *mix,const float *residual,const float *scale,const float *base,int M,int H,float eps){
@@ -688,18 +648,12 @@ extern "C" int dsv4_cuda_init(const int *dev,int n){if(n<1||n>16)return 0;float
return 1;}
static void plan_free(TcPlan*p){if(p->d)cublasLtMatrixLayoutDestroy(p->d);if(p->c)cublasLtMatrixLayoutDestroy(p->c);if(p->b)cublasLtMatrixLayoutDestroy(p->b);if(p->a)cublasLtMatrixLayoutDestroy(p->a);if(p->op)cublasLtMatmulDescDestroy(p->op);memset(p,0,sizeof(*p));}
extern "C" void dsv4_cuda_shutdown(void){
-#ifdef COLI_DSV4_FLASHINFER
- dsv4_flashinfer_shutdown();
-#endif
for(int i=0;i<ng;i++){cudaSetDevice(g[i].id);if(g[i].decode_state)cudaFree(g[i].decode_state);
#ifdef COLI_DSV4_NCCL
if(g[i].ep_comm)ncclCommDestroy(g[i].ep_comm);
#endif
}
for(int i=0;i<ng;i++){cudaSetDevice(g[i].id);if(g[i].dx)cudaFree(g[i].dx);if(g[i].dy)cudaFree(g[i].dy);if(g[i].p1)cudaFree(g[i].p1);if(g[i].p2)cudaFree(g[i].p2);if(g[i].p3)cudaFree(g[i].p3);if(g[i].p4)cudaFree(g[i].p4);if(g[i].aux1)cudaFree(g[i].aux1);if(g[i].aux2)cudaFree(g[i].aux2);if(g[i].aux3)cudaFree(g[i].aux3);if(g[i].mvdesc)cudaFree(g[i].mvdesc);if(g[i].expert_weights)cudaFree(g[i].expert_weights);if(g[i].expert_ids)cudaFree(g[i].expert_ids);if(g[i].tcw)cudaFree(g[i].tcw);if(g[i].tcs)cudaFree(g[i].tcs);if(g[i].tcx)cudaFree(g[i].tcx);if(g[i].tcxs)cudaFree(g[i].tcxs);if(g[i].tc_workspace)cudaFree(g[i].tc_workspace);
-#ifdef COLI_DSV4_VLLM_MHC
- if(g[i].mhcb1)cudaFree(g[i].mhcb1);if(g[i].mhcb2)cudaFree(g[i].mhcb2);if(g[i].mhcb3)cudaFree(g[i].mhcb3);
-#endif
#ifdef COLI_DSV4_DEEPGEMM
if(g[i].dga1)cudaFree(g[i].dga1);if(g[i].dga2)cudaFree(g[i].dga2);if(g[i].dgsfa1)cudaFree(g[i].dgsfa1);if(g[i].dgsfa2)cudaFree(g[i].dgsfa2);if(g[i].dglayout)cudaFree(g[i].dglayout);if(g[i].dgmm1)cudaFree(g[i].dgmm1);if(g[i].dgmm2)cudaFree(g[i].dgmm2);if(g[i].dgref)cudaFree(g[i].dgref);if(g[i].dgdensews)cudaFree(g[i].dgdensews);
#endif
@@ -748,9 +702,6 @@ static int upload(Dsv4CudaTensor **pt,const void *w,size_t wb,const uint8_t *sc,
extern "C" int dsv4_cuda_upload_fp8(Dsv4CudaTensor **t,const uint8_t*w,const uint8_t*s,int O,int I,int d){return upload(t,w,(size_t)O*I,s,(size_t)((O+127)/128)*((I+127)/128),O,I,d,8);}
extern "C" int dsv4_cuda_upload_fp8_bf16(Dsv4CudaTensor **t,const uint8_t*w,const uint8_t*s,int O,int I,int d){return upload(t,w,(size_t)O*I,s,(size_t)((O+127)/128)*((I+127)/128),O,I,d,9);}
extern "C" int dsv4_cuda_upload_fp4(Dsv4CudaTensor **t,const uint8_t*w,const uint8_t*s,int O,int I,int d){if(!upload(t,w,(size_t)O*I/2,s,(size_t)O*I/32,O,I,d,4))return 0;
-#ifdef COLI_DSV4_FLASHINFER
- size_t bytes=(size_t)((O+127)&~127)*((I/32+3)&~3);if(!ok(cudaMalloc(&(*t)->fi_scale,bytes),"FlashInfer scale allocation")||!ok(cudaMemset((*t)->fi_scale,0,bytes),"FlashInfer scale clear"))return 0;fi_swizzle_scale<<<((long long)O*(I/32)+255)/256,256>>>((*t)->fi_scale,(*t)->scale,O,I/32);if(!ok(cudaGetLastError(),"FlashInfer scale swizzle"))return 0;(*t)->bytes+=bytes;
-#endif
return 1;}
extern "C" int dsv4_cuda_upload_bf16(Dsv4CudaTensor **t,const uint16_t*w,int O,int I,int d){return upload(t,w,(size_t)O*I*2,nullptr,0,O,I,d,16);}
extern "C" int dsv4_cuda_upload_f32(Dsv4CudaTensor **t,const float*w,int O,int I,int d){return upload(t,w,(size_t)O*I*4,nullptr,0,O,I,d,32);}
@@ -828,18 +779,6 @@ extern "C" Dsv4CudaKvCache *dsv4_cuda_kv_create(int device, int window, int dim,
dsv4_cuda_kv_free(k);
return nullptr;
}
-#ifdef COLI_DSV4_FLASHINFER
- size_t fib=(size_t)((max_tokens+63)/64)*64*584,fic=(size_t)((k->max_compressed+63)/64)*64*584;
- k->fi_splits=(window+63)/64+(k->max_compressed+63)/64;
- if(!ok(cudaMalloc(&k->fi_kv,fib),"FlashInfer KV allocation")||!ok(cudaMemset(k->fi_kv,0,fib),"FlashInfer KV clear")||
- !ok(cudaMalloc(&k->fi_comp,fic),"FlashInfer compressed KV allocation")||!ok(cudaMemset(k->fi_comp,0,fic),"FlashInfer compressed KV clear")||
- !ok(cudaMalloc(&k->fi_indices,512*sizeof(int)),"FlashInfer indices allocation")||!ok(cudaMalloc(&k->fi_comp_indices,512*sizeof(int)),"FlashInfer compressed indices allocation")||!ok(cudaMalloc(&k->fi_length,sizeof(int)),"FlashInfer length allocation")||!ok(cudaMalloc(&k->fi_comp_length,sizeof(int)),"FlashInfer compressed length allocation")||
- !ok(cudaMalloc(&k->fi_q,64*512*sizeof(__nv_bfloat16)),"FlashInfer Q allocation")||!ok(cudaMalloc(&k->fi_mid,(size_t)64*k->fi_splits*512*sizeof(__nv_bfloat16)),"FlashInfer split output allocation")||
- !ok(cudaMalloc(&k->fi_lse,(size_t)64*k->fi_splits*sizeof(float)),"FlashInfer split LSE allocation")||!ok(cudaMalloc(&k->fi_out,64*512*sizeof(__nv_bfloat16)),"FlashInfer output allocation")||
- !ok(cudaMalloc(&k->fi_out_lse,64*sizeof(float)),"FlashInfer output LSE allocation")||!ok(cudaMalloc(&k->fi_slot,sizeof(int64_t)),"vLLM slot allocation")||
- !ok(cudaMalloc(&k->fi_pos,sizeof(int64_t)),"vLLM position allocation")||!ok(cudaMalloc(&k->fi_cos_sin,(size_t)max_tokens*64*sizeof(float)),"vLLM RoPE cache allocation")){dsv4_cuda_kv_free(k);return nullptr;}
- fi_cos_sin<<<(max_tokens*rope_pairs+255)/256,256,0,c->stream>>>(k->fi_cos_sin,k->rope_cos,k->rope_sin,max_tokens*rope_pairs);
-#endif
fill_value<<<(pb / 4 + 255) / 256, 256, 0, c->stream>>>(k->comp_score, pb / 4, -INFINITY);
if (!ok(cudaGetLastError(), "compress score clear")) {
dsv4_cuda_kv_free(k);
@@ -878,43 +817,10 @@ extern "C" int dsv4_cuda_mhc_post_pre(const Dsv4CudaActivation*x,const Dsv4CudaA
}
extern "C" int dsv4_cuda_mhc_post_pre_norm(const Dsv4CudaActivation*x,const Dsv4CudaActivation*r,Dsv4CudaActivation*state,int M,int H,Dsv4CudaActivation*out,Dsv4CudaTensor*fn,Dsv4CudaTensor*scale,Dsv4CudaTensor*base,Dsv4CudaTensor*norm,float rms_eps,float pre_eps,float sink_eps,float post_mult,int sink_iters,float norm_eps,Dsv4CudaActivation*input){
constexpr int N=24,parts=8;int I=M*H;Dev*c=x?ctx(x->device):nullptr;if(!c||!r||!state||!out||!fn||!scale||!base||!norm||!input||M!=4||r->device!=x->device||state->device!=x->device||out->device!=x->device||fn->device!=x->device||scale->device!=x->device||base->device!=x->device||norm->device!=x->device||input->device!=x->device||x->elements<H||r->elements<I||state->elements<M+M*M+M||out->elements<I||fn->fmt!=32||norm->fmt!=32||fn->O!=N||fn->I!=I||norm->O*norm->I<H||!ok(cudaSetDevice(x->device),"select fused normalized mHC device")||!buf((void**)&c->p1,&c->p1cap,N*sizeof(float))||!buf((void**)&c->dy,&c->ycap,(N+1)*parts*sizeof(float)))return 0;
-#ifdef COLI_DSV4_VLLM_MHC
- int vllm=getenv("DSV4_CUDA_VLLM_MHC")&&atoi(getenv("DSV4_CUDA_VLLM_MHC"));
- if(vllm&&H==4096&&rms_eps==1e-6f&&pre_eps==1e-6f&&sink_eps==1e-6f&&post_mult==2.f&&sink_iters==20&&norm_eps==1e-6f){
- if(!buf((void**)&c->mhcb1,&c->mhcb1cap,(size_t)I*sizeof(__nv_bfloat16))||!buf((void**)&c->mhcb2,&c->mhcb2cap,(size_t)H*sizeof(__nv_bfloat16))||!buf((void**)&c->mhcb3,&c->mhcb3cap,(size_t)I*sizeof(__nv_bfloat16)))return 0;
- if(!norm->bf16_cache){if(!ok(cudaMalloc(&norm->bf16_cache,(size_t)H*sizeof(__nv_bfloat16)),"mHC norm BF16 cache allocation"))return 0;mhc_float_to_bf16<<<(H+255)/256,256,0,c->stream>>>((__nv_bfloat16*)norm->bf16_cache,(float*)norm->w,H);}
- mhc_float_to_bf16<<<(I+255)/256,256,0,c->stream>>>(c->mhcb1,r->data,I);mhc_float_to_bf16<<<(H+255)/256,256,0,c->stream>>>(c->mhcb2,x->data,H);
- cudaError_t e=dsv4_vllm_mhc_post_pre_norm(c->stream,state->data+M,c->mhcb1,state->data,c->mhcb2,(float*)fn->w,c->dy,c->dy+N*parts,c->mhcb3,(float*)scale->w,(float*)base->w,c->mhcb2,norm->bf16_cache);
- if(e!=cudaSuccess)return ok(e,"vLLM mHC launch");mhc_bf16_to_float<<<(I+255)/256,256,0,c->stream>>>(out->data,c->mhcb3,I);mhc_bf16_to_float<<<(H+255)/256,256,0,c->stream>>>(input->data,c->mhcb2,H);return ok(cudaGetLastError(),"vLLM mHC conversion launch");
- }
-#endif
mhc_post_fn_split<<<parts,256,0,c->stream>>>((float*)fn->w,x->data,r->data,state->data,out->data,c->dy,N,M,H,parts);mhc_fn_finish<<<1,32,0,c->stream>>>(c->p1,c->dy,(float*)scale->w,(float*)base->w,N,M,I,rms_eps,parts);mhc_coeff<<<1,32,0,c->stream>>>(state->data,c->p1,M,pre_eps,sink_eps,post_mult,sink_iters);mhc_input_norm<<<1,256,0,c->stream>>>(input->data,out->data,state->data,(float*)norm->w,M,H,norm_eps);return ok(cudaGetLastError(),"fused normalized mHC post/pre launch");
}
extern "C" int dsv4_cuda_mhc_post_pre_norm_batch(const Dsv4CudaActivation*x,const Dsv4CudaActivation*r,Dsv4CudaActivation*state,int tokens,int H,Dsv4CudaActivation*out,Dsv4CudaTensor*fn,Dsv4CudaTensor*scale,Dsv4CudaTensor*base,Dsv4CudaTensor*norm,Dsv4CudaActivation*input){
-#ifdef COLI_DSV4_VLLM_MHC
- constexpr int M=4,N=24,parts=8;long long RH=(long long)tokens*M*H,XH=(long long)tokens*H;
- Dev*c=x?ctx(x->device):nullptr;if(!c||!r||!state||!out||!fn||!scale||!base||!norm||!input||tokens<1||H!=4096||RH>INT_MAX||XH>INT_MAX||
- r->device!=x->device||state->device!=x->device||out->device!=x->device||fn->device!=x->device||
- scale->device!=x->device||base->device!=x->device||norm->device!=x->device||input->device!=x->device||
- x->elements<XH||r->elements<RH||state->elements<(long long)tokens*(M+M*M)||out->elements<RH||input->elements<XH||
- fn->fmt!=32||norm->fmt!=32||fn->O!=N||fn->I!=M*H||norm->O*norm->I<H||
- !ok(cudaSetDevice(x->device),"select batched mHC device")||
- !buf((void**)&c->mhcb1,&c->mhcb1cap,(size_t)RH*sizeof(__nv_bfloat16))||
- !buf((void**)&c->mhcb2,&c->mhcb2cap,(size_t)XH*sizeof(__nv_bfloat16))||
- !buf((void**)&c->mhcb3,&c->mhcb3cap,(size_t)RH*sizeof(__nv_bfloat16))||
- !buf((void**)&c->dy,&c->ycap,(size_t)tokens*parts*(N+1)*sizeof(float)))return 0;
- if(!norm->bf16_cache){if(!ok(cudaMalloc(&norm->bf16_cache,(size_t)H*sizeof(__nv_bfloat16)),"mHC norm BF16 cache allocation"))return 0;mhc_float_to_bf16<<<(H+255)/256,256,0,c->stream>>>((__nv_bfloat16*)norm->bf16_cache,(float*)norm->w,H);}
- mhc_float_to_bf16<<<(RH+255)/256,256,0,c->stream>>>(c->mhcb1,r->data,(int)RH);
- mhc_float_to_bf16<<<(XH+255)/256,256,0,c->stream>>>(c->mhcb2,x->data,(int)XH);
- float*post=state->data,*comb=post+(long long)tokens*M,*mul=c->dy,*sq=mul+(long long)tokens*parts*N;
- cudaError_t e=dsv4_vllm_mhc_post_pre_norm_batch(c->stream,comb,c->mhcb1,post,c->mhcb2,(float*)fn->w,mul,sq,c->mhcb3,(float*)scale->w,(float*)base->w,c->mhcb2,norm->bf16_cache,tokens);
- if(e!=cudaSuccess)return ok(e,"vLLM batched mHC launch");
- mhc_bf16_to_float<<<(RH+255)/256,256,0,c->stream>>>(out->data,c->mhcb3,(int)RH);
- mhc_bf16_to_float<<<(XH+255)/256,256,0,c->stream>>>(input->data,c->mhcb2,(int)XH);
- return ok(cudaGetLastError(),"vLLM batched mHC conversion launch");
-#else
(void)x;(void)r;(void)state;(void)tokens;(void)H;(void)out;(void)fn;(void)scale;(void)base;(void)norm;(void)input;return 0;
-#endif
}
extern "C" int dsv4_cuda_mhc_post_batch(const Dsv4CudaActivation*x,const Dsv4CudaActivation*r,const Dsv4CudaActivation*state,int tokens,int H,Dsv4CudaActivation*out){
long long RH=(long long)tokens*4*H,XH=(long long)tokens*H;Dev*c=x?ctx(x->device):nullptr;if(!c||!r||!state||!out||tokens<1||H!=4096||r->device!=x->device||state->device!=x->device||out->device!=x->device||x->elements<XH||r->elements<RH||state->elements<(long long)tokens*20||out->elements<RH||!ok(cudaSetDevice(x->device),"select batched mHC output device"))return 0;mhc_output_batch<<<(RH+255)/256,256,0,c->stream>>>(out->data,x->data,r->data,state->data,state->data+(long long)tokens*4,H);return ok(cudaGetLastError(),"batched mHC output launch");
@@ -931,9 +837,6 @@ extern "C" void dsv4_cuda_kv_free(Dsv4CudaKvCache *k) {
if (k->rope_sin) cudaFree(k->rope_sin);
if (k->comp_cos) cudaFree(k->comp_cos);
if (k->comp_sin) cudaFree(k->comp_sin);
-#ifdef COLI_DSV4_FLASHINFER
- cudaFree(k->fi_kv);cudaFree(k->fi_comp);cudaFree(k->fi_indices);cudaFree(k->fi_comp_indices);cudaFree(k->fi_length);cudaFree(k->fi_comp_length);cudaFree(k->fi_q);cudaFree(k->fi_mid);cudaFree(k->fi_lse);cudaFree(k->fi_out);cudaFree(k->fi_out_lse);cudaFree(k->fi_slot);cudaFree(k->fi_pos);cudaFree(k->fi_cos_sin);
-#endif
free(k);
}
extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4CudaTensor *an, Dsv4CudaTensor *qa,
@@ -991,15 +894,9 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4C
if (qk_rope)
rope_heads_dynamic<<<(qk_rope / 2 + 255) / 256, 256, 0, comp>>>(
comp3, 1, dim, qk_rope, c->decode_state, 1 - ratio, 0, ratio, cache->comp_cos, cache->comp_sin);
- /* The FlashInfer cache packer performs the model's UE8M0 FP8
- * quantization. Quantizing and dequantizing here first would apply
- * a second, differently-scaled FP8 round before the real cache
- * store. Keep the simulation only for the float-cache fallback. */
-#ifndef COLI_DSV4_FLASHINFER
if (dim > qk_rope)
fp8_sim64_dynamic<<<(dim - qk_rope + 63) / 64, 64, 0, comp>>>(comp3, dim - qk_rope,
c->decode_state, ratio);
-#endif
compressed_store_dynamic<<<(dim + 255) / 256, 256, 0, comp>>>(
cache->compressed, comp3, cache->compressed_len, c->decode_state, ratio, cache->max_compressed, dim);
int commit_blocks = overlap ? (ratio * O + 255) / 256 : 1;
@@ -1022,12 +919,8 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4C
{fp8_sim<<<(R + 127) / 128, 128, 0, c->stream>>>(c->p1, 1, R);
run_mv<8>((uint8_t *)qb->w, qb->scale, c->p1, c->p2, Q, R, 1, c->stream);}
bf16_round<<<(Q + 255) / 256, 256, 0, c->stream>>>(c->p2, Q);
- int sparse=0;
-#ifdef COLI_DSV4_FLASHINFER
- sparse=getenv("DSV4_CUDA_SPARSE_MLA")&&atoi(getenv("DSV4_CUDA_SPARSE_MLA"))&&(heads==8||heads==16||heads==32||heads==64)&&dim==512&&cache->max_compressed<=512;
-#endif
- if(!sparse)head_rmsnorm<<<heads, 256, 0, c->stream>>>(c->p2, heads, dim, eps);
- if (qk_rope&&!sparse)
+ head_rmsnorm<<<heads, 256, 0, c->stream>>>(c->p2, heads, dim, eps);
+ if (qk_rope)
rope_heads_dynamic<<<(heads * (qk_rope / 2) + 255) / 256, 256, 0, c->stream>>>(
c->p2, heads, dim, qk_rope, c->decode_state, 0, 0, 0, cache->rope_cos, cache->rope_sin);
#ifdef COLI_DSV4_DEEPGEMM
@@ -1036,25 +929,12 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4C
run_mv<8>((uint8_t *)kv->w, kv->scale, c->dx, c->p3, K, H, 1, c->stream);
if (!fused_qkv) bf16_round<<<(K + 255) / 256, 256, 0, c->stream>>>(c->p3, K);
rmsnorm_f32<<<1, 256, 0, c->stream>>>(c->p3, fused_qkv ? c->p1 + R : c->p3, (float *)kvn->w, K, eps);
- if (qk_rope&&!sparse)
+ if (qk_rope)
rope_heads_dynamic<<<(qk_rope / 2 + 255) / 256, 256, 0, c->stream>>>(
c->p3, 1, dim, qk_rope, c->decode_state, 0, 0, 0, cache->rope_cos, cache->rope_sin);
- if(!sparse){if (K > qk_rope) fp8_sim64<<<(K - qk_rope + 63) / 64, 64, 0, c->stream>>>(c->p3, K - qk_rope);cache_store_dynamic<<<(K + 255) / 256, 256, 0, c->stream>>>(cache->kv, c->p3, c->decode_state, cache->window, K);}
-#ifdef COLI_DSV4_FLASHINFER
- else{fi_prepare_q<<<(heads*dim+255)/256,256,0,c->stream>>>(cache->fi_mid,c->p2,heads*dim);fi_prepare_q<<<(dim+255)/256,256,0,c->stream>>>(cache->fi_out,c->p3,dim);fi_position<<<1,1,0,c->stream>>>(cache->fi_slot,cache->fi_pos,c->decode_state);
- if(!dsv4_vllm_qnorm_rope_kv_insert(cache->fi_mid,cache->fi_q,cache->fi_out,cache->fi_kv,cache->fi_slot,cache->fi_pos,cache->fi_cos_sin,eps,heads,64,64*584,c->stream))return 0;}
-#endif
+ if (K > qk_rope) fp8_sim64<<<(K - qk_rope + 63) / 64, 64, 0, c->stream>>>(c->p3, K - qk_rope);
+ cache_store_dynamic<<<(K + 255) / 256, 256, 0, c->stream>>>(cache->kv, c->p3, c->decode_state, cache->window, K);
if (async && !ok(cudaStreamWaitEvent(c->stream, c->join, 0), "attention join wait")) return 0;
-#ifdef COLI_DSV4_FLASHINFER
- if(sparse){int main_topk=((cache->window+63)/64)*64,main_splits=main_topk/64;
- int use_extra=ratio&&(pos+1)>=ratio;
- int extra_cap=use_extra?(cache->max_tokens+ratio-1)/ratio:0,extra_topk=((extra_cap+63)/64)*64,extra_splits=extra_topk/64;
- if(use_extra&&!dsv4_vllm_pack_compressed(cache->compressed,cache->fi_comp,c->decode_state,ratio,64,64*584,c->stream))return 0;
- fi_indices<<<1,512,0,c->stream>>>(cache->fi_indices,cache->fi_length,c->decode_state,cache->window);if(use_extra)fi_extra_indices<<<1,512,0,c->stream>>>(cache->fi_comp_indices,cache->fi_comp_length,cache->compressed_len,c->decode_state,cache->window,ratio);
- if(!dsv4_flashinfer_sparse_mla(cache->fi_q,cache->fi_kv,cache->fi_indices,cache->fi_mid,cache->fi_lse,cache->fi_out,cache->fi_out_lse,cache->fi_length,(float*)sink->w,use_extra?cache->fi_comp:nullptr,use_extra?cache->fi_comp_indices:nullptr,use_extra?cache->fi_comp_length:nullptr,extra_topk,use_extra?64:0,use_extra?64*584:0,heads,main_topk,1,main_splits+extra_splits,0,1.f/sqrtf((float)dim),64*584,c->stream))return 0;
- mhc_bf16_to_float<<<(heads*dim+255)/256,256,0,c->stream>>>(c->p4,cache->fi_out,heads*dim);
- }else
-#endif
attention_window_dynamic<<<heads, 256, (size_t)(cache->window + cache->max_compressed) * 4, c->stream>>>(
c->p4, c->p2, cache->kv, cache->compressed, (float *)sink->w, heads, dim, cache->window, c->decode_state,
cache->compressed_len);
@@ -1075,55 +955,8 @@ extern "C" int dsv4_cuda_attention_window(const Dsv4CudaActivation *input, Dsv4C
return ok(cudaGetLastError(), "attention window launch");
}
-static int fi_reserve_batch(Dsv4CudaKvCache*k,int tokens,int heads,int topk,int splits){
-#ifdef COLI_DSV4_FLASHINFER
- size_t q=(size_t)tokens*heads*512*sizeof(__nv_bfloat16),mid=(size_t)tokens*heads*splits*512*sizeof(__nv_bfloat16),
- lse=(size_t)tokens*heads*splits*sizeof(float),out_lse=(size_t)tokens*heads*sizeof(float),
- indices=(size_t)tokens*topk*sizeof(int),length=(size_t)tokens*sizeof(int),pos=(size_t)tokens*sizeof(int64_t);
- return buf((void**)&k->fi_q,&k->fi_qcap,q)&&buf((void**)&k->fi_mid,&k->fi_midcap,mid)&&
- buf((void**)&k->fi_out,&k->fi_outcap,q)&&buf((void**)&k->fi_lse,&k->fi_lsecap,lse)&&
- buf((void**)&k->fi_out_lse,&k->fi_out_lsecap,out_lse)&&buf((void**)&k->fi_indices,&k->fi_indicescap,indices)&&
- buf((void**)&k->fi_length,&k->fi_lengthcap,length)&&buf((void**)&k->fi_slot,&k->fi_slotcap,pos)&&
- buf((void**)&k->fi_pos,&k->fi_poscap,pos);
-#else
- (void)k;(void)tokens;(void)heads;(void)topk;(void)splits;return 0;
-#endif
-}
-
extern "C" int dsv4_cuda_attention_sparse_batch(const Dsv4CudaActivation*input,Dsv4CudaTensor*an,Dsv4CudaTensor*qkv,Dsv4CudaTensor*qn,Dsv4CudaTensor*qb,Dsv4CudaTensor*kvn,Dsv4CudaTensor*sink,int heads,int dim,int start,int tokens,float eps,Dsv4CudaKvCache*cache,Dsv4CudaActivation*context){
-#if defined(COLI_DSV4_FLASHINFER) && defined(COLI_DSV4_DEEPGEMM)
-#define BATCH_ATTN_REQUIRE(expr,stage) do{if(!(expr)){fprintf(stderr,"[DSV4 batch attention] %s failed\n",stage);return 0;}}while(0)
- Dev*c=input?ctx(input->device):nullptr;int H=qkv?qkv->I:0,K=dim,R=qkv?qkv->O-K:0,Q=heads*dim,topk=cache?max(128,((cache->window+63)/64)*64):0,splits=topk/64;
- if(!c||!cache||!context||!an||!qkv||!qn||!qb||!kvn||!sink||tokens<1||start<0||start+tokens>cache->max_tokens||dim!=512||topk<64||topk>512||
- cache->device!=input->device||context->device!=input->device||input->elements<(long long)tokens*H||context->elements<(long long)tokens*Q||
- an->fmt!=32||qkv->fmt!=8||qn->fmt!=16||qb->fmt!=8||kvn->fmt!=32||sink->fmt!=32||qkv->O!=R+K||qb->I!=R||qb->O!=Q||
- an->O*an->I<H||qn->O*qn->I<R||kvn->O*kvn->I<K||sink->O*sink->I<heads){fprintf(stderr,"[DSV4 batch attention] validation failed\n");return 0;}
- Dsv4CudaTensor*tensors[]={an,qkv,qn,qb,kvn,sink};for(auto*t:tensors)if(t->device!=input->device){fprintf(stderr,"[DSV4 batch attention] device mismatch\n");return 0;}
- size_t nh=(size_t)tokens*H,nr=(size_t)tokens*R,nk=(size_t)tokens*K,nq=(size_t)tokens*Q;
- if(!ok(cudaSetDevice(input->device),"select batched sparse attention device")||!buf((void**)&c->dx,&c->xcap,nh*sizeof(float))||
- !buf((void**)&c->p1,&c->p1cap,(size_t)tokens*(R+K)*sizeof(float))||!buf((void**)&c->p2,&c->p2cap,nr*sizeof(float))||
- !buf((void**)&c->p3,&c->p3cap,nk*sizeof(float))||!buf((void**)&c->p4,&c->p4cap,nq*sizeof(float))||
- !buf((void**)&c->mhcb1,&c->mhcb1cap,nq*sizeof(__nv_bfloat16))||!buf((void**)&c->mhcb2,&c->mhcb2cap,nk*sizeof(__nv_bfloat16))||
- !fi_reserve_batch(cache,tokens,heads,topk,splits)){fprintf(stderr,"[DSV4 batch attention] workspace allocation failed\n");return 0;}
- rmsnorm_rows<<<tokens,256,0,c->stream>>>(c->dx,input->data,(float*)an->w,tokens,H,eps);
- BATCH_ATTN_REQUIRE(dg_dense_batch_dispatch(c,qkv,c->dx,c->p1,tokens),"QKV projection");
- split_qkv_rows<<<((long long)tokens*(R+K)+255)/256,256,0,c->stream>>>(c->p2,c->p3,c->p1,tokens,R,K);
- rmsnorm_rows_bf16<<<tokens,256,0,c->stream>>>(c->p2,c->p2,(__nv_bfloat16*)qn->w,tokens,R,eps);
- BATCH_ATTN_REQUIRE(dg_dense_batch_dispatch(c,qb,c->p2,c->p4,tokens),"Q projection");
- rmsnorm_rows<<<tokens,256,0,c->stream>>>(c->p3,c->p3,(float*)kvn->w,tokens,K,eps);
- mhc_float_to_bf16<<<(nq+255)/256,256,0,c->stream>>>(c->mhcb1,c->p4,(int)nq);
- mhc_float_to_bf16<<<(nk+255)/256,256,0,c->stream>>>(c->mhcb2,c->p3,(int)nk);
- fi_position_batch<<<(tokens+255)/256,256,0,c->stream>>>(cache->fi_slot,cache->fi_pos,start,tokens);
- BATCH_ATTN_REQUIRE(dsv4_vllm_qnorm_rope_kv_insert_batch(c->mhcb1,cache->fi_q,c->mhcb2,cache->fi_kv,cache->fi_slot,cache->fi_pos,cache->fi_cos_sin,eps,tokens,tokens,heads,heads,64,64*584,c->stream),"QNorm/RoPE/cache insert");
- fi_indices_batch<<<tokens,topk,0,c->stream>>>(cache->fi_indices,cache->fi_length,start,tokens,cache->window,topk);
- BATCH_ATTN_REQUIRE(dsv4_flashinfer_sparse_mla(cache->fi_q,cache->fi_kv,cache->fi_indices,cache->fi_mid,cache->fi_lse,cache->fi_out,cache->fi_out_lse,cache->fi_length,(float*)sink->w,nullptr,nullptr,nullptr,0,0,0,heads,topk,tokens,splits,0,1.f/sqrtf((float)dim),64*584,c->stream),"FlashInfer sparse MLA");
- mhc_bf16_to_float<<<(nq+255)/256,256,0,c->stream>>>(context->data,cache->fi_out,(int)nq);
- {int good=ok(cudaGetLastError(),"batched sparse attention launch");
-#undef BATCH_ATTN_REQUIRE
- return good;}
-#else
(void)input;(void)an;(void)qkv;(void)qn;(void)qb;(void)kvn;(void)sink;(void)heads;(void)dim;(void)start;(void)tokens;(void)eps;(void)cache;(void)context;return 0;
-#endif
}
extern "C" int dsv4_cuda_attention_output_batch(const Dsv4CudaActivation*context,Dsv4CudaTensor*wa,Dsv4CudaTensor*wb,int groups,int tokens,Dsv4CudaActivation*output){
@@ -1405,7 +1238,7 @@ extern "C" Dsv4CudaExpertSet *dsv4_cuda_expert_set_create(Dsv4CudaTensor *const
for(int e=0;e<count;e++){
Dsv4CudaTensor*t[]={gate[e],up[e],down[e]};if(!t[0]||!t[1]||!t[2]||t[0]->device!=device||t[1]->device!=device||t[2]->device!=device||
t[0]->fmt!=4||t[1]->fmt!=4||t[2]->fmt!=4||t[0]->I!=H||t[0]->O!=I||t[1]->I!=H||t[1]->O!=I||t[2]->I!=I||t[2]->O!=H){free(host);return nullptr;}
- host[e]={(uint8_t*)t[0]->w,t[0]->scale,t[0]->fi_scale};host[count+e]={(uint8_t*)t[1]->w,t[1]->scale,t[1]->fi_scale};host[2*count+e]={(uint8_t*)t[2]->w,t[2]->scale,t[2]->fi_scale};
+ host[e]={(uint8_t*)t[0]->w,t[0]->scale};host[count+e]={(uint8_t*)t[1]->w,t[1]->scale};host[2*count+e]={(uint8_t*)t[2]->w,t[2]->scale};
}
Dsv4CudaExpertSet *set=(Dsv4CudaExpertSet*)calloc(1,sizeof(*set));if(!set){free(host);return nullptr;}
set->count=count;set->device=device;set->H=H;set->I=I;set->sg=sg;set->su=su;set->sd=sd;
@@ -1559,18 +1392,10 @@ extern "C" int dsv4_cuda_route_moe(const Dsv4CudaActivation*input,Dsv4CudaTensor
#endif
fp8_sim<<<(H+127)/128,128,0,c->stream>>>(c->dx,1,H);
build_moe_desc<<<1,K,0,c->stream>>>(c->mvdesc,set->table,c->expert_ids,c->dx,c->p1,c->p2,c->p3,c->p4,K,set->count,H,I);
-#ifdef COLI_DSV4_FLASHINFER
- if(!dsv4_flashinfer_grouped(c->mvdesc,2*K,I,H,input->device,c->stream))return 0;
-#else
- run_fp4_grouped(c->mvdesc,2*K,I,H,c->stream);
-#endif
+run_fp4_grouped(c->mvdesc,2*K,I,H,c->stream);
expert_act_grouped<<<(K*I+255)/256,256,0,c->stream>>>(c->p3,c->p1,c->p2,c->expert_weights,limit,K,I);
fp8_sim<<<K*((I+127)/128),128,0,c->stream>>>(c->p3,K,I);
-#ifdef COLI_DSV4_FLASHINFER
- if(!dsv4_flashinfer_grouped(c->mvdesc+2*K,K,H,I,input->device,c->stream))return 0;
-#else
- run_fp4_grouped(c->mvdesc+2*K,K,H,I,c->stream);
-#endif
+run_fp4_grouped(c->mvdesc+2*K,K,H,I,c->stream);
expert_reduce<<<(H+255)/256,256,0,c->stream>>>(output->data,c->p4,K,H);
run_mv<8>((uint8_t*)set->sg->w,set->sg->scale,c->dx,c->p1,I,H,1,c->stream);
run_mv<8>((uint8_t*)set->su->w,set->su->scale,c->dx,c->p2,I,H,1,c->stream);
@@ -1715,5 +1540,5 @@ extern "C" int dsv4_cuda_wo(Dsv4CudaTensor *wa,Dsv4CudaTensor *wb,int groups,flo
bf16_round<<<(H+255)/256,256,0,c->stream>>>(c->p2,H);
return ok(cudaGetLastError(),"wo launch")&&ok(cudaMemcpyAsync(out,c->p2,hb,cudaMemcpyDeviceToHost,c->stream),"wo result download")&&ok(cudaStreamSynchronize(c->stream),"wo sync");
}
-extern "C" void dsv4_cuda_tensor_free(Dsv4CudaTensor*t){if(!t)return;cudaSetDevice(t->device);if(t->own_w&&t->w)cudaFree(t->w);if(t->bf16_cache)cudaFree(t->bf16_cache);if(t->own_scale&&t->scale)cudaFree(t->scale);if(t->own_dg_scale&&t->dg_scale)cudaFree(t->dg_scale);if(t->fi_scale)cudaFree(t->fi_scale);free(t);}
+extern "C" void dsv4_cuda_tensor_free(Dsv4CudaTensor*t){if(!t)return;cudaSetDevice(t->device);if(t->own_w&&t->w)cudaFree(t->w);if(t->bf16_cache)cudaFree(t->bf16_cache);if(t->own_scale&&t->scale)cudaFree(t->scale);if(t->own_dg_scale&&t->dg_scale)cudaFree(t->dg_scale);free(t);}
extern "C" long long dsv4_cuda_tensor_bytes(const Dsv4CudaTensor*t){return t?t->bytes:0;}
diff --git a/c/backend_cuda_dsv4_flashinfer.cu b/c/backend_cuda_dsv4_flashinfer.cu
deleted file mode 100644
index 2b7e5e8e..00000000
--- a/c/backend_cuda_dsv4_flashinfer.cu
+++ /dev/null
@@ -1,71 +0,0 @@
-#include "backend_cuda_dsv4_flashinfer.h"
-#include <cuda_bf16.h>
-#include <cuda_fp8.h>
-#include <flashinfer/gemm/group_gemm_mxfp4_groupwise_sm120.cuh>
-#include <flashinfer/attention/sparse_mla_sm120/model/kv_cache_traits.cuh>
-#include <cstdio>
-#include <cstdlib>
-
-namespace flashinfer::group_gemm {
-template <bool SwapAB,int ScaleGranularity,typename ScaleConfig,typename ElementA,typename ElementB,
- typename ElementSFA,typename ElementSFB,typename ElementD,typename ProblemShape,typename StrideA,
- typename StrideB,typename StrideD,typename LayoutSFA,typename LayoutSFB>
-__global__ void compute_pointer_group_gemm_args(
- ElementA *packed_a,ElementB *B,ElementSFA *packed_sfa,ElementSFB *SFB,ElementD *D,int *m_indptr,
- int n,int k,int num_groups,ProblemShape *problem_sizes,const ElementA **A_ptr,const ElementB **B_ptr,
- const ElementSFA **SFA_ptr,const ElementSFB **SFB_ptr,ElementD **D_ptr,StrideA *stride_A,
- StrideB *stride_B,StrideD *stride_D,LayoutSFA *layout_SFA,LayoutSFB *layout_SFB){
- int i=blockIdx.x*blockDim.x+threadIdx.x;if(i>=num_groups)return;
- auto A_list=reinterpret_cast<const ElementA *const *>(packed_a);
- auto SFA_list=reinterpret_cast<const ElementSFA *const *>(packed_sfa);
- constexpr size_t mn_align=128,k_align=(size_t)ScaleGranularity*4;
- size_t sf_n=((size_t)n+mn_align-1)/mn_align*mn_align,swk=((size_t)k+k_align-1)/k_align*k_align,sf_k=swk/ScaleGranularity;
- asm volatile("griddepcontrol.wait;");asm volatile("griddepcontrol.launch_dependents;");
- int mo=m_indptr[i],mn=m_indptr[i+1];size_t m=(size_t)(mn-mo),sf_mo=((size_t)mo+(size_t)i*(mn_align-1))/mn_align*mn_align;
- static_assert(SwapAB,"pointer grouped GEMM is decode-only");
- problem_sizes[i]=ProblemShape(n,m,k);stride_A[i]=cutlass::make_cute_packed_stride(StrideA{},{n,k,1});
- stride_B[i]=cutlass::make_cute_packed_stride(StrideB{},{m,k,1});stride_D[i]=cutlass::make_cute_packed_stride(StrideD{},{n,m,1});
- A_ptr[i]=A_list[i];B_ptr[i]=B+(size_t)mo*k;D_ptr[i]=D+(size_t)mo*n;
- layout_SFA[i]=ScaleConfig::tile_atom_to_shape_SFA(make_shape((int)sf_n,(int)m,(int)swk,1));SFA_ptr[i]=SFA_list[i];
- layout_SFB[i]=ScaleConfig::tile_atom_to_shape_SFB(make_shape((int)sf_n,(int)m,(int)swk,1));SFB_ptr[i]=SFB+sf_mo*sf_k;
-}
-#define compute_sm120_cutlass_group_gemm_args compute_pointer_group_gemm_args
-INSTANTIATE_GROUP_GEMM_MXFP4_GROUPWISE_SM120(
- 128,32,128,true,cutlass::float_e4m3_t,cutlass::float_e2m1_t,
- cutlass::float_ue8m0_t,cutlass::float_ue8m0_t,cutlass::bfloat16_t,
- fp8,fp4,ue8m0,ue8m0,bf16)
-#undef compute_sm120_cutlass_group_gemm_args
-}
-using namespace flashinfer::group_gemm;
-
-namespace flashinfer::sparse_mla_sm120 {
-bool launch_sparse_mla_decode_dsv4(ModelType,int,int,int,int,int,const bf16*,const uint8_t*,
- const int32_t*,bf16*,float*,bf16*,float*,const int*,
- const float*,const uint8_t*,const int32_t*,const int*,int,
- int,size_t,int,float,size_t,cudaStream_t);
-}
-
-struct Desc { const uint8_t *w,*scale;const float *x;float *y; };
-struct FiCtx { int device=-1;const uint8_t **wp=nullptr,**sp=nullptr;uint8_t *ws=nullptr,*a=nullptr,*as=nullptr;__nv_bfloat16 *out=nullptr;int *indptr=nullptr;void *iw=nullptr,*fw=nullptr;cudaGraph_t graph=nullptr;cudaGraphExec_t exec=nullptr;size_t wpc=0,spc=0,wsc=0,ac=0,asc=0,oc=0,ic=0; };
-static FiCtx fc[16][2];
-static int grow(void **p,size_t *cap,size_t n){if(*cap>=n)return 1;if(*p)cudaFree(*p);*p=nullptr;*cap=0;if(cudaMalloc(p,n)!=cudaSuccess)return 0;*cap=n;return 1;}
-__device__ __forceinline__ int sf_off(int row,int col,int cols){return (col&3)+(col/4)*512+(row&31)*16+((row&127)/32)*4+(row/128)*128*((cols+3)&~3);}
-__global__ void stage_pointer(const Desc*d,const uint8_t**wp,const uint8_t**sp,int count){int t=threadIdx.x;if(t<count){wp[t]=d[t].w;sp[t]=d[t].scale;}}
-__global__ void quant_activation(const Desc*d,uint8_t*a,uint8_t*s,int count,int I){int g=blockIdx.x/(I/32),b=blockIdx.x%(I/32),i=b*32+threadIdx.x;float v=d[g].x[i],mx=fabsf(v);for(int m=16;m;m>>=1)mx=fmaxf(mx,__shfl_xor_sync(0xffffffff,mx,m));float raw=fmaxf(mx,1e-4f)/448.f;int e;frexpf(raw,&e);float sc=ldexpf(1.f,e-1);if(sc<raw)sc*=2;int se;frexpf(sc,&se);uint8_t q=__nv_cvt_float_to_fp8(v/sc,__NV_SATFINITE,__NV_E4M3);for(int r=0;r<4;r++){a[((long long)g*4+r)*I+i]=q;if(!threadIdx.x)s[(long long)g*128*((I/32+3)&~3)+sf_off(r,b,I/32)]=(uint8_t)(se-1+127);}}
-__global__ void make_indptr(int*p,int count){int i=threadIdx.x;if(i<=count)p[i]=i*4;}
-__global__ void extract(const Desc*d,const __nv_bfloat16*out,int count,int O){int p=blockIdx.x*blockDim.x+threadIdx.x;if(p<count*O){int g=p/O,o=p%O;d[g].y[o]=__bfloat162float(out[(long long)g*4*O+o]);}}
-
-extern "C" int dsv4_flashinfer_grouped(const void *descriptors,int count,int O,int I,int device,cudaStream_t stream){if(!descriptors||count<1||count>16||O%128||I%128||device<0||device>=16)return 0;if(cudaSetDevice(device)!=cudaSuccess)return 0;FiCtx&c=fc[device][O>I];c.device=device;size_t sb=(size_t)count*O*(I/32),ab=(size_t)count*4*I,asb=(size_t)count*128*((I/32+3)&~3),ob=(size_t)count*4*O*2;
- if(!grow((void**)&c.wp,&c.wpc,count*sizeof(*c.wp))||!grow((void**)&c.sp,&c.spc,count*sizeof(*c.sp))||!grow((void**)&c.ws,&c.wsc,sb)||!grow((void**)&c.a,&c.ac,ab)||!grow((void**)&c.as,&c.asc,asb)||!grow((void**)&c.out,&c.oc,ob)||!grow((void**)&c.indptr,&c.ic,(count+1)*4)||(!c.iw&&cudaMalloc(&c.iw,8<<20)!=cudaSuccess)||(!c.fw&&cudaMalloc(&c.fw,64<<20)!=cudaSuccess))return 0;
- if(c.exec)return cudaGraphLaunch(c.exec,stream)==cudaSuccess;
- auto launch=[&]()->int{cudaMemsetAsync(c.as,0,asb,stream);stage_pointer<<<1,32,0,stream>>>((const Desc*)descriptors,c.wp,c.sp,count);quant_activation<<<count*(I/32),32,0,stream>>>((const Desc*)descriptors,c.a,c.as,count,I);make_indptr<<<1,32,0,stream>>>(c.indptr,count);cudaError_t e=CutlassMXFP4GroupwiseScaledGroupGEMMSM120<128,32,128,true,cutlass::float_e4m3_t,cutlass::float_e2m1_t,cutlass::float_ue8m0_t,cutlass::float_ue8m0_t,cutlass::bfloat16_t>(c.iw,8<<20,c.fw,64<<20,(cutlass::float_e4m3_t*)c.a,(cutlass::float_e2m1_t*)c.wp,(cutlass::float_ue8m0_t*)c.as,(cutlass::float_ue8m0_t*)c.sp,(cutlass::bfloat16_t*)c.out,c.indptr,O,I,count,stream,device);if(e!=cudaSuccess)return 0;extract<<<(count*O+255)/256,256,0,stream>>>((const Desc*)descriptors,c.out,count,O);return cudaGetLastError()==cudaSuccess;};
- static int graph=-1;if(graph<0){const char*p=getenv("DSV4_CUDA_FLASHINFER_GRAPH");graph=p&&atoi(p)!=0;}if(!graph)return launch();if(c.exec)return cudaGraphLaunch(c.exec,stream)==cudaSuccess;if(!launch()||cudaStreamSynchronize(stream)!=cudaSuccess||cudaStreamBeginCapture(stream,cudaStreamCaptureModeThreadLocal)!=cudaSuccess||!launch()||cudaStreamEndCapture(stream,&c.graph)!=cudaSuccess||cudaGraphInstantiate(&c.exec,c.graph,0)!=cudaSuccess)return 0;return 1;}
-extern "C" int dsv4_flashinfer_sparse_mla(const void*q,const void*kv,const int*indices,void*mid,float*mid_lse,void*out,float*out_lse,const int*length,const float*sink,const void*extra_kv,const int*extra_indices,const int*extra_length,int extra_topk,int pbs_extra,size_t extra_stride,int heads,int topk,int tokens,int splits,int chunks,float scale,size_t stride,cudaStream_t stream){
- using namespace flashinfer::sparse_mla_sm120;
- if(!q||!kv||!indices||!mid||!mid_lse||!out||!length||heads<1||tokens<1||splits<1)return 0;
- return launch_sparse_mla_decode_dsv4(ModelType::DSV4,heads,topk,64,tokens,splits,
- (const bf16*)q,(const uint8_t*)kv,(const int32_t*)indices,(bf16*)mid,mid_lse,
- (bf16*)out,out_lse,length,sink,(const uint8_t*)extra_kv,(const int32_t*)extra_indices,
- extra_length,extra_topk,pbs_extra,extra_stride,chunks,scale,stride,stream);
-}
-extern "C" void dsv4_flashinfer_shutdown(void){for(auto&dev:fc)for(auto&c:dev)if(c.device>=0){cudaSetDevice(c.device);if(c.exec)cudaGraphExecDestroy(c.exec);if(c.graph)cudaGraphDestroy(c.graph);cudaFree(c.wp);cudaFree(c.sp);cudaFree(c.ws);cudaFree(c.a);cudaFree(c.as);cudaFree(c.out);cudaFree(c.indptr);cudaFree(c.iw);cudaFree(c.fw);c=FiCtx{};}}
diff --git a/c/backend_cuda_dsv4_flashinfer.h b/c/backend_cuda_dsv4_flashinfer.h
deleted file mode 100644
index fd66632f..00000000
--- a/c/backend_cuda_dsv4_flashinfer.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#ifndef COLIBRI_BACKEND_CUDA_DSV4_FLASHINFER_H
-#define COLIBRI_BACKEND_CUDA_DSV4_FLASHINFER_H
-#include <cuda_runtime.h>
-#ifdef __cplusplus
-extern "C" {
-#endif
-int dsv4_flashinfer_grouped(const void *descriptors,int count,int O,int I,int device,cudaStream_t stream);
-int dsv4_flashinfer_sparse_mla(const void *q,const void *kv_cache,const int *indices,
- void *mid_out,float *mid_lse,void *output,float *out_lse,
- const int *topk_length,const float *attn_sink,
- const void *extra_kv_cache,const int *extra_indices,
- const int *extra_topk_length,int extra_topk,int pbs_extra,
- size_t stride_extra_kv_block,int heads,int topk,int tokens,
- int splits,int chunks_per_block,float sm_scale,
- size_t stride_kv_block,cudaStream_t stream);
-void dsv4_flashinfer_shutdown(void);
-#ifdef __cplusplus
-}
-#endif
-#endif
diff --git a/c/backend_cuda_dsv4_mhc_vllm.cu b/c/backend_cuda_dsv4_mhc_vllm.cu
deleted file mode 100644
index 1a1e1ed8..00000000
--- a/c/backend_cuda_dsv4_mhc_vllm.cu
+++ /dev/null
@@ -1,535 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// Generated by vLLM TileLang at ffd46bfab2128bb84146050e98b51a617c6575ab
-#include <tl_templates/cuda/gemm.h>
-#include <tl_templates/cuda/copy.h>
-#include <tl_templates/cuda/reduce.h>
-#include <tl_templates/cuda/ldsm.h>
-#include <tl_templates/cuda/threadblock_swizzle.h>
-#include <tl_templates/cuda/debug.h>
-#ifdef ENABLE_BF16
-#include <tl_templates/cuda/cuda_bf16_fallbacks.cuh>
-#endif
-
-extern "C" __global__ void mhc_fused_tilelang_kernel(const float* comb_mix, const float* post_mix, const bfloat16_t* residual_in, bfloat16_t* residual_out, float* rp_out, const float* weight_t, const bfloat16_t* x_in, float* yp_out, int num_tokens, int split_k);
-extern "C" __global__ void __launch_bounds__(256, 1) mhc_fused_tilelang_kernel(const float* comb_mix, const float* post_mix, const bfloat16_t* residual_in, bfloat16_t* residual_out, float* rp_out, const float* weight_t, const bfloat16_t* x_in, float* yp_out, int num_tokens, int split_k) {
- extern __shared__ __align__(1024) uchar buf_dyn_shmem[];
- float acc[2];
- float sqr[1];
- float pm[4];
- float cm[16];
- float new_r[4];
- float v = 0x0p+0f/*0.000000e+00*/;
- float v2 = 0x0p+0f/*0.000000e+00*/;
- float broadcast_var = 0x0p+0f/*0.000000e+00*/;
- *(float2*)(acc + 0) = make_float2(broadcast_var, broadcast_var);
- sqr[0] = 0x0p+0f/*0.000000e+00*/;
- cudaGridDependencySynchronize();
- if (((int)threadIdx.x) < 4) {
- ((float*)buf_dyn_shmem)[((int)threadIdx.x)] = post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))];
- }
- if (((int)threadIdx.x) < 16) {
- ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] = comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + ((int64_t)((int)threadIdx.x)))];
- }
- __syncthreads();
- #pragma unroll
- for (int j = 0; j < 4; ++j) {
- pm[j] = ((float*)buf_dyn_shmem)[j];
- }
- #pragma unroll
- for (int j_1 = 0; j_1 < 4; ++j_1) {
- #pragma unroll
- for (int k = 0; k < 4; ++k) {
- cm[((k * 4) + j_1)] = ((float*)buf_dyn_shmem)[(((k * 4) + j_1) + 4)];
- }
- }
- for (int it = 0; it < ((4096 / split_k) >> 8); ++it) {
- #pragma unroll
- for (int j_2 = 0; j_2 < 4; ++j_2) {
- new_r[j_2] = (pm[j_2] * ((float)x_in[((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))]));
- #pragma unroll
- for (int k_1 = 0; k_1 < 4; ++k_1) {
- new_r[j_2] = (new_r[j_2] + (cm[((k_1 * 4) + j_2)] * ((float)residual_in[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)k_1) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))])));
- }
- }
- if (((int)blockIdx.y) == 0) {
- #pragma unroll
- for (int j_3 = 0; j_3 < 4; ++j_3) {
- residual_out[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + (((int64_t)j_3) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))] = ((bfloat16_t)new_r[j_3]);
- sqr[0] = (sqr[0] + (new_r[j_3] * new_r[j_3]));
- }
- }
- #pragma unroll
- for (int n = 0; n < 2; ++n) {
- #pragma unroll
- for (int j_4 = 0; j_4 < 4; ++j_4) {
- acc[n] = (acc[n] + (weight_t[((((((((int64_t)((int)blockIdx.y)) * (int64_t)32768) + (((int64_t)n) * (int64_t)16384)) + (((int64_t)j_4) * (int64_t)4096)) + (((int64_t)it) * (int64_t)256)) + (((int64_t)((int)blockIdx.z)) * ((int64_t)4096 / ((int64_t)split_k)))) + ((int64_t)((int)threadIdx.x)))] * new_r[j_4]));
- }
- }
- }
- #pragma unroll
- for (int n_1 = 0; n_1 < 2; ++n_1) {
- acc[n_1] = tl::warp_reduce_sum(acc[n_1]);
- }
- if (((int)blockIdx.y) == 0) {
- sqr[0] = tl::warp_reduce_sum(sqr[0]);
- }
- if ((((int)threadIdx.x) % 32) == 0) {
- #pragma unroll
- for (int n_2 = 0; n_2 < 2; ++n_2) {
- ((float*)buf_dyn_shmem)[((((((int)threadIdx.x) >> 5) * 3) + n_2) + 20)] = acc[n_2];
- }
- if (((int)blockIdx.y) == 0) {
- ((float*)buf_dyn_shmem)[(((((int)threadIdx.x) >> 5) * 3) + 22)] = sqr[0];
- }
- }
- __syncthreads();
- if ((((int)threadIdx.x) >> 5) == 0) {
- if ((((int)threadIdx.x) & 31) < 2) {
- #pragma unroll
- for (int w = 0; w < 8; ++w) {
- v = (v + ((float*)buf_dyn_shmem)[(((w * 3) + (((int)threadIdx.x) & 31)) + 20)]);
- }
- yp_out[((((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)((int)blockIdx.z)) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)blockIdx.y)) * (int64_t)2)) + (((int64_t)((int)threadIdx.x)) & (int64_t)31))] = v;
- }
- if ((((int)blockIdx.y) == 0) && ((((int)threadIdx.x) % 32) == 0)) {
- #pragma unroll
- for (int w_1 = 0; w_1 < 8; ++w_1) {
- v2 = (v2 + ((float*)buf_dyn_shmem)[((w_1 * 3) + 22)]);
- }
- rp_out[((((int64_t)((int)blockIdx.z)) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))] = v2;
- }
- }
-cudaTriggerProgrammaticLaunchCompletion();
-}
-
-extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(
- float *, const float *, const float *, const float *, const float *,
- bfloat16_t *, const bfloat16_t *, float *, const bfloat16_t *, int);
-
-extern "C" cudaError_t dsv4_vllm_mhc_post_pre_norm_batch(
- cudaStream_t stream, const float *comb_mix, const void *residual_in,
- const float *post_mix, const void *x_in, const float *weight_t,
- float *gemm_out_mul, float *gemm_out_sqrsum, void *residual_out,
- const float *hc_scale, const float *hc_base, void *layer_input,
- const void *norm_weight, int num_tokens) {
- if (num_tokens < 1) return cudaErrorInvalidValue;
- const int split_k = 8;
- mhc_fused_tilelang_kernel<<<dim3(num_tokens, 12, split_k), 256, 176, stream>>>(
- comb_mix, post_mix, static_cast<const bfloat16_t *>(residual_in),
- static_cast<bfloat16_t *>(residual_out), gemm_out_sqrsum, weight_t,
- static_cast<const bfloat16_t *>(x_in), gemm_out_mul, num_tokens,
- split_k);
- cudaError_t error = cudaGetLastError();
- if (error != cudaSuccess) return error;
- mhc_pre_big_fuse_with_norm_tilelang_kernel<<<num_tokens, 96, 29040, stream>>>(
- const_cast<float *>(comb_mix), gemm_out_mul, gemm_out_sqrsum, hc_base,
- hc_scale, static_cast<bfloat16_t *>(layer_input),
- static_cast<const bfloat16_t *>(norm_weight),
- const_cast<float *>(post_mix),
- static_cast<const bfloat16_t *>(residual_out), num_tokens);
- return cudaGetLastError();
-}
-
-extern "C" cudaError_t dsv4_vllm_mhc_post_pre_norm(
- cudaStream_t stream, const float *comb_mix, const void *residual_in,
- const float *post_mix, const void *x_in, const float *weight_t,
- float *gemm_out_mul, float *gemm_out_sqrsum, void *residual_out,
- const float *hc_scale, const float *hc_base, void *layer_input,
- const void *norm_weight) {
- return dsv4_vllm_mhc_post_pre_norm_batch(
- stream, comb_mix, residual_in, post_mix, x_in, weight_t, gemm_out_mul,
- gemm_out_sqrsum, residual_out, hc_scale, hc_base, layer_input,
- norm_weight, 1);
-}
-
-
-#include <math_constants.h>
-#include <tl_templates/cuda/gemm.h>
-#include <tl_templates/cuda/copy.h>
-#include <tl_templates/cuda/reduce.h>
-#include <tl_templates/cuda/ldsm.h>
-#include <tl_templates/cuda/threadblock_swizzle.h>
-#include <tl_templates/cuda/debug.h>
-#ifdef ENABLE_BF16
-#include <tl_templates/cuda/cuda_bf16_fallbacks.cuh>
-#endif
-
-extern "C" __global__ void mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens);
-extern "C" __global__ void __launch_bounds__(96, 1) mhc_pre_big_fuse_with_norm_tilelang_kernel(float* comb_mix, const float* gemm_out_mul, const float* gemm_out_sqrsum, const float* hc_base, const float* hc_scale, bfloat16_t* layer_input, const bfloat16_t* norm_weight, float* post_mix, const bfloat16_t* residual, int num_tokens) {
- extern __shared__ __align__(1024) uchar buf_dyn_shmem[];
- float mixes[1];
- float rms[1];
- float cm[1];
- float row_max[1];
- float row_sum[1];
- float col_sum[1];
- float sumsq_per_pos[16];
- float sumsq[1];
- float rsqrt_norm[1];
- float xl[64];
- float ol[16];
- bfloat16_t xs_local_cast[8];
- bfloat16_t xs_local_cast_1[8];
- bfloat16_t xs_local_cast_2[8];
- float w_local[16];
- float ol_1[16];
- bfloat16_t w_shared_local_cast_3[8];
- bfloat16_t output_shared_local_cast_4[8];
- bfloat16_t layer_input_local_cast_5[8];
- bfloat16_t w_shared_local_cast_6[8];
- bfloat16_t output_shared_local_cast_7[8];
- bfloat16_t layer_input_local_cast_8[8];
- bfloat16_t w_shared_local_cast_9[8];
- bfloat16_t output_shared_local_cast_10[8];
- bfloat16_t layer_input_local_cast_11[8];
- mixes[0] = 0x0p+0f/*0.000000e+00*/;
- rms[0] = 0x0p+0f/*0.000000e+00*/;
- cudaGridDependencySynchronize();
- for (int i_split = 0; i_split < 8; ++i_split) {
- rms[0] = (rms[0] + gemm_out_sqrsum[((((int64_t)i_split) * ((int64_t)num_tokens)) + ((int64_t)((int)blockIdx.x)))]);
- }
- rms[0] = rsqrtf(((rms[0] / 0x1p+14f/*1.638400e+04*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));
- mixes[0] = 0x0p+0f/*0.000000e+00*/;
- for (int i_split_1 = 0; i_split_1 < 8; ++i_split_1) {
- mixes[0] = (mixes[0] + gemm_out_mul[(((((int64_t)((int)blockIdx.x)) * (int64_t)24) + ((((int64_t)i_split_1) * ((int64_t)num_tokens)) * (int64_t)24)) + (((int64_t)((int)threadIdx.x)) % (int64_t)24))]);
- }
- mixes[0] = (mixes[0] * rms[0]);
- if ((((int)threadIdx.x) / 24) == 0) {
- ((float*)buf_dyn_shmem)[(((int)threadIdx.x) % 24)] = mixes[0];
- }
- __syncthreads();
- if (((int)threadIdx.x) < 32) {
- if (((int)threadIdx.x) < 4) {
- post_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)4) + ((int64_t)((int)threadIdx.x)))] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 4)] * hc_scale[1]) + hc_base[(((int)threadIdx.x) + 4)]))))) * 0x1p+1f/*2.000000e+00*/);
- }
- cm[0] = ((((float*)buf_dyn_shmem)[((((int)threadIdx.x) & 15) + 8)] * hc_scale[2]) + hc_base[((((int)threadIdx.x) & 15) + 8)]);
- row_max[0] = -CUDART_INF_F;
- row_max[0] = max(row_max[0], cm[0]);
- row_max[0] = tl::AllReduce<tl::MaxOp, 4, 1, 0, tl::NamedBarrier<32>>::run(row_max[0]);
- cm[0] = expf((cm[0] - row_max[0]));
- row_sum[0] = 0x0p+0f/*0.000000e+00*/;
- row_sum[0] = (row_sum[0] + cm[0]);
- row_sum[0] = tl::AllReduce<tl::SumOp, 4, 1, 0, tl::NamedBarrier<32>>::run(row_sum[0]);
- cm[0] = ((cm[0] / row_sum[0]) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/);
- col_sum[0] = 0x0p+0f/*0.000000e+00*/;
- col_sum[0] = (col_sum[0] + cm[0]);
- col_sum[0] = tl::AllReduce<tl::SumOp, 16, 4, 0, tl::NamedBarrier<32>>::run(col_sum[0]);
- cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));
- for (int __1 = 0; __1 < 19; ++__1) {
- row_sum[0] = 0x0p+0f/*0.000000e+00*/;
- row_sum[0] = (row_sum[0] + cm[0]);
- row_sum[0] = tl::AllReduce<tl::SumOp, 4, 1, 0, tl::NamedBarrier<32>>::run(row_sum[0]);
- cm[0] = (cm[0] / (row_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));
- col_sum[0] = 0x0p+0f/*0.000000e+00*/;
- col_sum[0] = (col_sum[0] + cm[0]);
- col_sum[0] = tl::AllReduce<tl::SumOp, 16, 4, 0, tl::NamedBarrier<32>>::run(col_sum[0]);
- cm[0] = (cm[0] / (col_sum[0] + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));
- }
- if ((((int)threadIdx.x) >> 4) == 0) {
- comb_mix[((((int64_t)((int)blockIdx.x)) * (int64_t)16) + (((int64_t)((int)threadIdx.x)) & (int64_t)15))] = cm[0];
- }
- } else {
- if (((int)threadIdx.x) < 36) {
- ((float*)buf_dyn_shmem)[(((int)threadIdx.x) + 7224)] = ((0x1p+0f/*1.000000e+00*/ / (0x1p+0f/*1.000000e+00*/ + expf((0x0p+0f/*0.000000e+00*/ - ((((float*)buf_dyn_shmem)[(((int)threadIdx.x) - 32)] * hc_scale[0]) + hc_base[(((int)threadIdx.x) - 32)]))))) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/);
- }
- #pragma unroll
- for (int i = 0; i < 4; ++i) {
- float broadcast_var = 0x0p+0f/*0.000000e+00*/;
- *(float4*)(sumsq_per_pos + (i * 4)) = make_float4(broadcast_var, broadcast_var, broadcast_var, broadcast_var);
- }
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_1 = 0; i_1 < 8; ++i_1) {
- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_1 >> 1) * 1024) + (((((i_1 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_1) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_1) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8))])));
- }
- tl::cp_async_commit();
- #pragma unroll
- for (int i_2 = 0; i_2 < 8; ++i_2) {
- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i_2 >> 1) * 1024) + (((((i_2 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 4144)])), (&(residual[((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_2) >> (int64_t)1) * (int64_t)4096)) + (((((((int64_t)i_2) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)1024)])));
- }
- tl::cp_async_commit();
- for (int i0_h = 0; i0_h < 2; ++i0_h) {
- tl::cp_async_wait<1>();
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_3 = 0; i_3 < 8; ++i_3) {
- *(uint4*)(xs_local_cast + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h * 4096) + (i_3 * 512)) + (((int)threadIdx.x) * 8)) - 208));
- for (int vec = 0; vec < 2; ++vec) {
- float4 __2;
- uint2 v_ = *(uint2*)(xs_local_cast + (vec * 4));
- ((float2*)(&__2))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[0]);
- ((float2*)(&__2))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v_))[1]);
- *(float4*)(xl + ((i_3 * 8) + (vec * 4))) = __2;
- }
- }
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_4 = 0; i_4 < 8; ++i_4) {
- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h * 4096) + ((i_4 >> 1) * 1024)) + (((((i_4 * 64) + ((int)threadIdx.x)) + 96) & 127) * 8)) + 48)])), (&(residual[(((((((int64_t)((int)blockIdx.x)) * (int64_t)16384) + ((((int64_t)i_4) >> (int64_t)1) * (int64_t)4096)) + (((int64_t)i0_h) * (int64_t)1024)) + (((((((int64_t)i_4) * (int64_t)64) + ((int64_t)((int)threadIdx.x))) + (int64_t)96) & (int64_t)127) * (int64_t)8)) + (int64_t)2048)])));
- }
- tl::cp_async_commit();
- #pragma unroll
- for (int i_5 = 0; i_5 < 4; ++i_5) {
- float broadcast_var_1 = 0x0p+0f/*0.000000e+00*/;
- *(float4*)(ol + (i_5 * 4)) = make_float4(broadcast_var_1, broadcast_var_1, broadcast_var_1, broadcast_var_1);
- }
- tl::__sync_thread_partial<3, 64>();
- for (int i_hc = 0; i_hc < 4; ++i_hc) {
- float pre = ((float*)buf_dyn_shmem)[(i_hc + 7256)];
- #pragma unroll
- for (int i_6 = 0; i_6 < 16; ++i_6) {
- ol[i_6] = (ol[i_6] + (pre * xl[((i_hc * 16) + i_6)]));
- }
- }
- #pragma unroll
- for (int i_7 = 0; i_7 < 4; ++i_7) {
- float4 __3;
- float4 v__1 = *(float4*)(ol + (i_7 * 4));
- float4 v__2 = *(float4*)(sumsq_per_pos + (i_7 * 4));
- *(float2*)(&(__3.x)) = tl::fma2(*(float2*)(&(v__1.x)), *(float2*)(&(v__1.x)), *(float2*)(&(v__2.x)));
- *(float2*)(&(__3.z)) = tl::fma2(*(float2*)(&(v__1.z)), *(float2*)(&(v__1.z)), *(float2*)(&(v__2.z)));
- *(float4*)(sumsq_per_pos + (i_7 * 4)) = __3;
- uint2 __4;
- float4 v__3 = *(float4*)(ol + (i_7 * 4));
- (reinterpret_cast<__nv_bfloat162*>(&__4))[0] = __float22bfloat162_rn(((float2*)(&v__3))[0]);
- (reinterpret_cast<__nv_bfloat162*>(&__4))[1] = __float22bfloat162_rn(((float2*)(&v__3))[1]);
- *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i0_h * 1024) + ((i_7 >> 1) * 512)) + (((int)threadIdx.x) * 8)) + ((i_7 & 1) * 4)) + 7984)) = __4;
- }
- }
- tl::cp_async_wait<1>();
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_8 = 0; i_8 < 8; ++i_8) {
- *(uint4*)(xs_local_cast_1 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_8 * 512) + (((int)threadIdx.x) * 8)) - 208));
- for (int vec_1 = 0; vec_1 < 2; ++vec_1) {
- float4 __5;
- uint2 v__4 = *(uint2*)(xs_local_cast_1 + (vec_1 * 4));
- ((float2*)(&__5))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[0]);
- ((float2*)(&__5))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__4))[1]);
- *(float4*)(xl + ((i_8 * 8) + (vec_1 * 4))) = __5;
- }
- }
- #pragma unroll
- for (int i_9 = 0; i_9 < 4; ++i_9) {
- float broadcast_var_2 = 0x0p+0f/*0.000000e+00*/;
- *(float4*)(ol + (i_9 * 4)) = make_float4(broadcast_var_2, broadcast_var_2, broadcast_var_2, broadcast_var_2);
- }
- for (int i_hc_1 = 0; i_hc_1 < 4; ++i_hc_1) {
- float pre_1 = ((float*)buf_dyn_shmem)[(i_hc_1 + 7256)];
- #pragma unroll
- for (int i_10 = 0; i_10 < 16; ++i_10) {
- ol[i_10] = (ol[i_10] + (pre_1 * xl[((i_hc_1 * 16) + i_10)]));
- }
- }
- #pragma unroll
- for (int i_11 = 0; i_11 < 4; ++i_11) {
- float4 __6;
- float4 v__5 = *(float4*)(ol + (i_11 * 4));
- float4 v__6 = *(float4*)(sumsq_per_pos + (i_11 * 4));
- *(float2*)(&(__6.x)) = tl::fma2(*(float2*)(&(v__5.x)), *(float2*)(&(v__5.x)), *(float2*)(&(v__6.x)));
- *(float2*)(&(__6.z)) = tl::fma2(*(float2*)(&(v__5.z)), *(float2*)(&(v__5.z)), *(float2*)(&(v__6.z)));
- *(float4*)(sumsq_per_pos + (i_11 * 4)) = __6;
- uint2 __7;
- float4 v__7 = *(float4*)(ol + (i_11 * 4));
- (reinterpret_cast<__nv_bfloat162*>(&__7))[0] = __float22bfloat162_rn(((float2*)(&v__7))[0]);
- (reinterpret_cast<__nv_bfloat162*>(&__7))[1] = __float22bfloat162_rn(((float2*)(&v__7))[1]);
- *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_11 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_11 & 1) * 4)) + 10032)) = __7;
- }
- tl::cp_async_wait<0>();
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_12 = 0; i_12 < 8; ++i_12) {
- *(uint4*)(xs_local_cast_2 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_12 * 512) + (((int)threadIdx.x) * 8)) + 3888));
- for (int vec_2 = 0; vec_2 < 2; ++vec_2) {
- float4 __8;
- uint2 v__8 = *(uint2*)(xs_local_cast_2 + (vec_2 * 4));
- ((float2*)(&__8))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[0]);
- ((float2*)(&__8))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__8))[1]);
- *(float4*)(xl + ((i_12 * 8) + (vec_2 * 4))) = __8;
- }
- }
- #pragma unroll
- for (int i_13 = 0; i_13 < 4; ++i_13) {
- float broadcast_var_3 = 0x0p+0f/*0.000000e+00*/;
- *(float4*)(ol + (i_13 * 4)) = make_float4(broadcast_var_3, broadcast_var_3, broadcast_var_3, broadcast_var_3);
- }
- for (int i_hc_2 = 0; i_hc_2 < 4; ++i_hc_2) {
- float pre_2 = ((float*)buf_dyn_shmem)[(i_hc_2 + 7256)];
- #pragma unroll
- for (int i_14 = 0; i_14 < 16; ++i_14) {
- ol[i_14] = (ol[i_14] + (pre_2 * xl[((i_hc_2 * 16) + i_14)]));
- }
- }
- #pragma unroll
- for (int i_15 = 0; i_15 < 4; ++i_15) {
- float4 __9;
- float4 v__9 = *(float4*)(ol + (i_15 * 4));
- float4 v__10 = *(float4*)(sumsq_per_pos + (i_15 * 4));
- *(float2*)(&(__9.x)) = tl::fma2(*(float2*)(&(v__9.x)), *(float2*)(&(v__9.x)), *(float2*)(&(v__10.x)));
- *(float2*)(&(__9.z)) = tl::fma2(*(float2*)(&(v__9.z)), *(float2*)(&(v__9.z)), *(float2*)(&(v__10.z)));
- *(float4*)(sumsq_per_pos + (i_15 * 4)) = __9;
- uint2 __10;
- float4 v__11 = *(float4*)(ol + (i_15 * 4));
- (reinterpret_cast<__nv_bfloat162*>(&__10))[0] = __float22bfloat162_rn(((float2*)(&v__11))[0]);
- (reinterpret_cast<__nv_bfloat162*>(&__10))[1] = __float22bfloat162_rn(((float2*)(&v__11))[1]);
- *(uint2*)(((bfloat16_t*)buf_dyn_shmem) + (((((i_15 >> 1) * 512) + (((int)threadIdx.x) * 8)) + ((i_15 & 1) * 4)) + 11056)) = __10;
- }
- sumsq[0] = 0x0p+0f/*0.000000e+00*/;
- #pragma unroll
- for (int rv = 0; rv < 16; ++rv) {
- sumsq[0] = (sumsq[0] + sumsq_per_pos[(((rv & 1) * 8) + (rv >> 1))]);
- }
- tl::__sync_thread_partial<3, 64>();
- sumsq[0] = tl::AllReduce<tl::SumOp, 64, 1, 32, tl::NamedBarrier<64>>::run(sumsq[0], (&(((float*)buf_dyn_shmem)[7192])));
- rsqrt_norm[0] = rsqrtf(((sumsq[0] / 0x1p+12f/*4.096000e+03*/) + 0x1.0c6f7a0b5ed8dp-20f/*1.000000e-06*/));
- #pragma unroll
- for (int i_16 = 0; i_16 < 2; ++i_16) {
- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_16 * 512) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[(((i_16 * 512) + (((int)threadIdx.x) * 8)) - 256)])));
- }
- tl::cp_async_commit();
- #pragma unroll
- for (int i_17 = 0; i_17 < 2; ++i_17) {
- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 13104)])), (&(norm_weight[(((i_17 * 512) + (((int)threadIdx.x) * 8)) + 768)])));
- }
- tl::cp_async_commit();
- for (int i0_h_1 = 0; i0_h_1 < 2; ++i0_h_1) {
- tl::cp_async_wait<1>();
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_18 = 0; i_18 < 2; ++i_18) {
- *(uint4*)(w_shared_local_cast_3 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_18 * 512)) + (((int)threadIdx.x) * 8)) + 12080));
- for (int vec_3 = 0; vec_3 < 2; ++vec_3) {
- float4 __11;
- uint2 v__12 = *(uint2*)(w_shared_local_cast_3 + (vec_3 * 4));
- ((float2*)(&__11))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[0]);
- ((float2*)(&__11))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__12))[1]);
- *(float4*)(w_local + ((i_18 * 8) + (vec_3 * 4))) = __11;
- }
- }
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_19 = 0; i_19 < 2; ++i_19) {
- tl::cp_async_gs<16>((&(((bfloat16_t*)buf_dyn_shmem)[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 12080)])), (&(norm_weight[((((i0_h_1 * 1024) + (i_19 * 512)) + (((int)threadIdx.x) * 8)) + 1792)])));
- }
- tl::cp_async_commit();
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_20 = 0; i_20 < 2; ++i_20) {
- *(uint4*)(output_shared_local_cast_4 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + ((((i0_h_1 * 1024) + (i_20 * 512)) + (((int)threadIdx.x) * 8)) + 7984));
- for (int vec_4 = 0; vec_4 < 2; ++vec_4) {
- float4 __12;
- float4 __13;
- float4 __14;
- uint2 v__13 = *(uint2*)(output_shared_local_cast_4 + (vec_4 * 4));
- ((float2*)(&__14))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[0]);
- ((float2*)(&__14))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__13))[1]);
- float4 v__14 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]);
- *(float2*)(&(__13.x)) = tl::mul2(*(float2*)(&(__14.x)), *(float2*)(&(v__14.x)));
- *(float2*)(&(__13.z)) = tl::mul2(*(float2*)(&(__14.z)), *(float2*)(&(v__14.z)));
- float4 v__15 = *(float4*)(w_local + ((i_20 * 8) + (vec_4 * 4)));
- *(float2*)(&(__12.x)) = tl::mul2(*(float2*)(&(__13.x)), *(float2*)(&(v__15.x)));
- *(float2*)(&(__12.z)) = tl::mul2(*(float2*)(&(__13.z)), *(float2*)(&(v__15.z)));
- *(float4*)(ol_1 + ((i_20 * 8) + (vec_4 * 4))) = __12;
- }
- }
- #pragma unroll
- for (int i_21 = 0; i_21 < 2; ++i_21) {
- for (int vec_5 = 0; vec_5 < 2; ++vec_5) {
- uint2 __15;
- float4 v__16 = *(float4*)(ol_1 + ((i_21 * 8) + (vec_5 * 4)));
- (reinterpret_cast<__nv_bfloat162*>(&__15))[0] = __float22bfloat162_rn(((float2*)(&v__16))[0]);
- (reinterpret_cast<__nv_bfloat162*>(&__15))[1] = __float22bfloat162_rn(((float2*)(&v__16))[1]);
- *(uint2*)(layer_input_local_cast_5 + (vec_5 * 4)) = __15;
- }
- *(uint4*)(layer_input + (((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i0_h_1) * (int64_t)1024)) + (((int64_t)i_21) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) - (int64_t)256)) = *(uint4*)(layer_input_local_cast_5 + 0);
- }
- }
- tl::cp_async_wait<1>();
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_22 = 0; i_22 < 2; ++i_22) {
- *(uint4*)(w_shared_local_cast_6 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_22 * 512) + (((int)threadIdx.x) * 8)) + 12080));
- for (int vec_6 = 0; vec_6 < 2; ++vec_6) {
- float4 __16;
- uint2 v__17 = *(uint2*)(w_shared_local_cast_6 + (vec_6 * 4));
- ((float2*)(&__16))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[0]);
- ((float2*)(&__16))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__17))[1]);
- *(float4*)(w_local + ((i_22 * 8) + (vec_6 * 4))) = __16;
- }
- }
- #pragma unroll
- for (int i_23 = 0; i_23 < 2; ++i_23) {
- *(uint4*)(output_shared_local_cast_7 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_23 * 512) + (((int)threadIdx.x) * 8)) + 10032));
- for (int vec_7 = 0; vec_7 < 2; ++vec_7) {
- float4 __17;
- float4 __18;
- float4 __19;
- uint2 v__18 = *(uint2*)(output_shared_local_cast_7 + (vec_7 * 4));
- ((float2*)(&__19))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[0]);
- ((float2*)(&__19))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__18))[1]);
- float4 v__19 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]);
- *(float2*)(&(__18.x)) = tl::mul2(*(float2*)(&(__19.x)), *(float2*)(&(v__19.x)));
- *(float2*)(&(__18.z)) = tl::mul2(*(float2*)(&(__19.z)), *(float2*)(&(v__19.z)));
- float4 v__20 = *(float4*)(w_local + ((i_23 * 8) + (vec_7 * 4)));
- *(float2*)(&(__17.x)) = tl::mul2(*(float2*)(&(__18.x)), *(float2*)(&(v__20.x)));
- *(float2*)(&(__17.z)) = tl::mul2(*(float2*)(&(__18.z)), *(float2*)(&(v__20.z)));
- *(float4*)(ol_1 + ((i_23 * 8) + (vec_7 * 4))) = __17;
- }
- }
- #pragma unroll
- for (int i_24 = 0; i_24 < 2; ++i_24) {
- for (int vec_8 = 0; vec_8 < 2; ++vec_8) {
- uint2 __20;
- float4 v__21 = *(float4*)(ol_1 + ((i_24 * 8) + (vec_8 * 4)));
- (reinterpret_cast<__nv_bfloat162*>(&__20))[0] = __float22bfloat162_rn(((float2*)(&v__21))[0]);
- (reinterpret_cast<__nv_bfloat162*>(&__20))[1] = __float22bfloat162_rn(((float2*)(&v__21))[1]);
- *(uint2*)(layer_input_local_cast_8 + (vec_8 * 4)) = __20;
- }
- *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_24) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)1792)) = *(uint4*)(layer_input_local_cast_8 + 0);
- }
- tl::cp_async_wait<0>();
- tl::__sync_thread_partial<3, 64>();
- #pragma unroll
- for (int i_25 = 0; i_25 < 2; ++i_25) {
- *(uint4*)(w_shared_local_cast_9 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_25 * 512) + (((int)threadIdx.x) * 8)) + 13104));
- for (int vec_9 = 0; vec_9 < 2; ++vec_9) {
- float4 __21;
- uint2 v__22 = *(uint2*)(w_shared_local_cast_9 + (vec_9 * 4));
- ((float2*)(&__21))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[0]);
- ((float2*)(&__21))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__22))[1]);
- *(float4*)(w_local + ((i_25 * 8) + (vec_9 * 4))) = __21;
- }
- }
- #pragma unroll
- for (int i_26 = 0; i_26 < 2; ++i_26) {
- *(uint4*)(output_shared_local_cast_10 + 0) = *(uint4*)(((bfloat16_t*)buf_dyn_shmem) + (((i_26 * 512) + (((int)threadIdx.x) * 8)) + 11056));
- for (int vec_10 = 0; vec_10 < 2; ++vec_10) {
- float4 __22;
- float4 __23;
- float4 __24;
- uint2 v__23 = *(uint2*)(output_shared_local_cast_10 + (vec_10 * 4));
- ((float2*)(&__24))[0] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[0]);
- ((float2*)(&__24))[1] = __bfloat1622float2((reinterpret_cast<__nv_bfloat162*>(&v__23))[1]);
- float4 v__24 = make_float4(rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0], rsqrt_norm[0]);
- *(float2*)(&(__23.x)) = tl::mul2(*(float2*)(&(__24.x)), *(float2*)(&(v__24.x)));
- *(float2*)(&(__23.z)) = tl::mul2(*(float2*)(&(__24.z)), *(float2*)(&(v__24.z)));
- float4 v__25 = *(float4*)(w_local + ((i_26 * 8) + (vec_10 * 4)));
- *(float2*)(&(__22.x)) = tl::mul2(*(float2*)(&(__23.x)), *(float2*)(&(v__25.x)));
- *(float2*)(&(__22.z)) = tl::mul2(*(float2*)(&(__23.z)), *(float2*)(&(v__25.z)));
- *(float4*)(ol_1 + ((i_26 * 8) + (vec_10 * 4))) = __22;
- }
- }
- #pragma unroll
- for (int i_27 = 0; i_27 < 2; ++i_27) {
- for (int vec_11 = 0; vec_11 < 2; ++vec_11) {
- uint2 __25;
- float4 v__26 = *(float4*)(ol_1 + ((i_27 * 8) + (vec_11 * 4)));
- (reinterpret_cast<__nv_bfloat162*>(&__25))[0] = __float22bfloat162_rn(((float2*)(&v__26))[0]);
- (reinterpret_cast<__nv_bfloat162*>(&__25))[1] = __float22bfloat162_rn(((float2*)(&v__26))[1]);
- *(uint2*)(layer_input_local_cast_11 + (vec_11 * 4)) = __25;
- }
- *(uint4*)(layer_input + ((((((int64_t)((int)blockIdx.x)) * (int64_t)4096) + (((int64_t)i_27) * (int64_t)512)) + (((int64_t)((int)threadIdx.x)) * (int64_t)8)) + (int64_t)2816)) = *(uint4*)(layer_input_local_cast_11 + 0);
- }
- }
- cudaTriggerProgrammaticLaunchCompletion();
-}
diff --git a/c/backend_cuda_dsv4_mhc_vllm.h b/c/backend_cuda_dsv4_mhc_vllm.h
deleted file mode 100644
index 02ab41d9..00000000
--- a/c/backend_cuda_dsv4_mhc_vllm.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef COLIBRI_BACKEND_CUDA_DSV4_MHC_VLLM_H
-#define COLIBRI_BACKEND_CUDA_DSV4_MHC_VLLM_H
-
-#include <cuda_runtime_api.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-cudaError_t dsv4_vllm_mhc_post_pre_norm(
- cudaStream_t stream, const float *comb_mix, const void *residual_in,
- const float *post_mix, const void *x_in, const float *weight_t,
- float *gemm_out_mul, float *gemm_out_sqrsum, void *residual_out,
- const float *hc_scale, const float *hc_base, void *layer_input,
- const void *norm_weight);
-cudaError_t dsv4_vllm_mhc_post_pre_norm_batch(
- cudaStream_t stream, const float *comb_mix, const void *residual_in,
- const float *post_mix, const void *x_in, const float *weight_t,
- float *gemm_out_mul, float *gemm_out_sqrsum, void *residual_out,
- const float *hc_scale, const float *hc_base, void *layer_input,
- const void *norm_weight, int num_tokens);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/c/backend_cuda_dsv4_qkv_vllm.cu b/c/backend_cuda_dsv4_qkv_vllm.cu
deleted file mode 100644
index e43c90a0..00000000
--- a/c/backend_cuda_dsv4_qkv_vllm.cu
+++ /dev/null
@@ -1,739 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * SPDX-FileCopyrightText: Copyright contributors to the vLLM project
- *
- * Horizontally-fused DeepseekV4-MLA kernel:
- * - Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last ROPE_DIM
- * - KV side: GPT-J RoPE on last ROPE_DIM + UE8M0 FP8 quant on NoPE + paged
- * cache insert
- *
- * Structured after `applyMLARopeAndAssignQKVKernelGeneration` in
- * TensorRT-LLM's mlaKernels.cu: one kernel, one grid, with head-slot
- * dispatch choosing Q vs KV work per warp. The per-warp RMSNorm/RoPE
- * skeleton is adapted from vllm-deepseek_v4's existing
- * `fusedQKNormRopeKernel` (csrc/fused_qknorm_rope_kernel.cu).
- *
- * Assumptions (hard-coded for DeepseekV4 attention):
- * HEAD_DIM = 512
- * ROPE_DIM = 64 (RoPE applied to dims [NOPE_DIM, HEAD_DIM))
- * NOPE_DIM = 448
- * QUANT_BLOCK = 64 (UE8M0 FP8 quant block)
- * FP8_MAX = 224.0f on ROCm FNUZ / 448.0f on OCP
- * is_neox=false (GPT-J interleaved pairs)
- * cos_sin_cache layout [max_pos, rope_dim] = cos || sin (cos first, sin
- * second along last dim; each half is rope_dim/2 = 32 values)
- *
- * Cache layout per paged-cache block (block_size tokens):
- * [0, bs*576): token data, 448 fp8 + 128 bf16 each
- * [bs*576, bs*576 + bs*8): UE8M0 scales, 7 real + 1 pad per token
- */
-
-#include <cmath>
-#include <cuda_bf16.h>
-#include <cuda_fp8.h>
-#include <cuda_runtime.h>
-#include <cstdint>
-#include <type_traits>
-
-namespace vllm {
-template <typename T> struct _typeConvert;
-template <> struct _typeConvert<__nv_bfloat16> {
- using packed_hip_type = __nv_bfloat162;
- __device__ static float2 convert(__nv_bfloat162 v) { return __bfloat1622float2(v); }
- __device__ static __nv_bfloat162 convert(float2 v) { return __floats2bfloat162_rn(v.x, v.y); }
- __device__ static __nv_bfloat16 convert(float v) { return __float2bfloat16(v); }
- __device__ static float convert(__nv_bfloat16 v) { return __bfloat162float(v); }
-};
-}
-
-#ifndef FINAL_MASK
- #ifdef USE_ROCM
- #define FINAL_MASK 0xffffffffffffffffULL
- #else
- #define FINAL_MASK 0xffffffffu
- #endif
-#endif
-
-#ifdef USE_ROCM
-// ROCm-compatible FP8 conversion helpers
-__device__ __forceinline__ uint8_t rocm_cvt_float_to_fp8_e4m3(float val) {
- // gfx942 uses FNUZ FP8; other ROCm targets use OCP E4M3.
- #if defined(__gfx942__)
- __hip_fp8_e4m3_fnuz fp8_val(val);
- #else
- __hip_fp8_e4m3 fp8_val(val);
- #endif
- return reinterpret_cast<uint8_t&>(fp8_val);
-}
-#endif
-
-namespace vllm {
-namespace deepseek_v4_fused_ops {
-
-namespace {
-inline int getSMVersion() {
- int device = 0;
- cudaDeviceProp props{};
- cudaGetDevice(&device);
- cudaGetDeviceProperties(&props, device);
- return props.major * 10 + props.minor;
-}
-} // namespace
-
-// ────────────────────────────────────────────────────────────────────────────
-// Constants
-// ────────────────────────────────────────────────────────────────────────────
-constexpr int kHeadDim = 512;
-constexpr int kRopeDim = 64;
-constexpr int kNopeDim = kHeadDim - kRopeDim; // 448
-constexpr int kQuantBlock = 64;
-constexpr int kNumQuantBlocks = kNopeDim / kQuantBlock; // 7
-constexpr int kScaleBytesPerToken = kNumQuantBlocks + 1; // 8 (7 real + 1 pad)
-constexpr int kTokenDataBytes = kNopeDim + kRopeDim * 2; // 448 + 128 = 576
-// FNUZ on gfx942 / OCP elsewhere. FNUZ uses 224.0 (not the dtype's raw
-// 240.0) to match the rest of vLLM's FNUZ pipeline.
-#if defined(USE_ROCM) && defined(__gfx942__)
-constexpr float kFp8Max = 224.0f;
-#else
-constexpr float kFp8Max = 448.0f;
-#endif
-
-#ifndef USE_ROCM
-// When num_tokens is less than this threshold,
-// run the reduced grid variant on cuda
-constexpr float NUM_TOKEN_CUTOFF = 1024;
-#endif
-
-// Per-warp layout: 32 lanes Γ— 16 elems/lane = 512 elems = HEAD_DIM.
-constexpr int kNumLanes = 32;
-constexpr int kElemsPerLane = kHeadDim / kNumLanes; // 16
-
-// Pack this lane's 16 fp32 elements into per-tensor E4M3 FP8 (one uint4 = 16
-// B), scaling by `scale` (a reciprocal scale) and saturating to Β±448. Used by
-// the FlashInfer full-cache path for both the Q and KV stores.
-__device__ __forceinline__ uint4 packFp8E4M3x16(float const* values,
- float const scale) {
-#ifndef USE_ROCM
- uint4 out;
- auto* out2 = reinterpret_cast<__nv_fp8x2_storage_t*>(&out);
- #pragma unroll
- for (int i = 0; i < kElemsPerLane / 2; i++) {
- float2 scaled =
- make_float2(values[2 * i] * scale, values[2 * i + 1] * scale);
- scaled.x = fminf(fmaxf(scaled.x, -kFp8Max), kFp8Max);
- scaled.y = fminf(fmaxf(scaled.y, -kFp8Max), kFp8Max);
- out2[i] = __nv_cvt_float2_to_fp8x2(scaled, __NV_SATFINITE, __NV_E4M3);
- }
- return out;
-#else
- uint8_t out_bytes[kElemsPerLane];
- #pragma unroll
- for (int i = 0; i < kElemsPerLane; i++) {
- float scaled = values[i] * scale;
- scaled = fminf(fmaxf(scaled, -kFp8Max), kFp8Max);
- out_bytes[i] = rocm_cvt_float_to_fp8_e4m3(scaled);
- }
- return *reinterpret_cast<uint4 const*>(out_bytes);
-#endif
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-// Small inline helpers
-// ────────────────────────────────────────────────────────────────────────────
-__device__ __forceinline__ float warp4MaxAbs(float val) {
- // Reduce absolute max across 4 consecutive lanes (lane id & 3 group).
- float peer = __shfl_xor_sync(FINAL_MASK, val, 1);
- val = fmaxf(val, peer);
- peer = __shfl_xor_sync(FINAL_MASK, val, 2);
- val = fmaxf(val, peer);
- return val;
-}
-
-template <typename T>
-__device__ __forceinline__ float warpSum(float val) {
-#pragma unroll
- for (int mask = 16; mask > 0; mask >>= 1) {
- val += __shfl_xor_sync(FINAL_MASK, val, mask, 32);
- }
- return val;
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-// Per-slot inner pipeline
-// ────────────────────────────────────────────────────────────────────────────
-// Shared by both kernel variants: 1 CTA per (token, head) pair vs. 1 CTA per
-// token. Templated on `kNumHeadsQPadded` so the KV-sentinel comparison and
-// q_out stride fold to compile-time constants.
-//
-// Slot layout (per token):
-// slot < num_heads_q β†’ live-Q (RMSNorm + RoPE,
-// read q_in β†’
-// write q_out)
-// num_heads_q <= slot < kNumHeadsQPadded β†’ pad-Q (zero-fill q_out;
-// v0/v1 unused)
-// slot == kNumHeadsQPadded β†’ KV (RoPE + UE8M0 quant
-// + paged-cache
-// insert)
-template <typename scalar_t_in, int kNumHeadsQPadded>
-__device__ __forceinline__ void processDeepseekV4Slot(
- uint4 v0, uint4 v1, int const tokenIdx, int const slotIdx,
- int const dim_base, int const laneId, int const num_heads_q,
- float const eps, scalar_t_in* __restrict__ q_out,
- uint8_t* __restrict__ k_cache, int64_t const* __restrict__ slot_mapping,
- int64_t const* __restrict__ position_ids,
- float const* __restrict__ cos_sin_cache, int const cache_block_size,
- int const kv_block_stride) {
- using Converter = vllm::_typeConvert<scalar_t_in>;
- bool const isKV = (slotIdx == kNumHeadsQPadded);
- bool const isPadQ = !isKV && (slotIdx >= num_heads_q);
-
- // ── Pad-Q branch: write 32 B of zeros and exit. ─────────────────────────
- // FlashMLA reads these slots; bf16 +0.0 is bit pattern 0x0000, so a uint4
- // zero literal is correct. Matches the live-Q branch's vectorized store.
- if (isPadQ) {
- scalar_t_in* dst =
- q_out +
- (static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
- kHeadDim +
- dim_base;
- uint4 const zero4 = {0u, 0u, 0u, 0u};
- *reinterpret_cast<uint4*>(dst) = zero4;
- *reinterpret_cast<uint4*>(dst + 8) = zero4;
- return;
- }
-
- // ── Decode the bf16 β†’ 16 fp32 registers ─────────────────────────────
- float elements[kElemsPerLane];
- {
- typename Converter::packed_hip_type const* p0 =
- reinterpret_cast<typename Converter::packed_hip_type const*>(&v0);
- typename Converter::packed_hip_type const* p1 =
- reinterpret_cast<typename Converter::packed_hip_type const*>(&v1);
-#pragma unroll
- for (int i = 0; i < 4; i++) {
- float2 f2 = Converter::convert(p0[i]);
- elements[2 * i] = f2.x;
- elements[2 * i + 1] = f2.y;
- }
-#pragma unroll
- for (int i = 0; i < 4; i++) {
- float2 f2 = Converter::convert(p1[i]);
- elements[8 + 2 * i] = f2.x;
- elements[8 + 2 * i + 1] = f2.y;
- }
- }
-
- // ── Q branch: RMSNorm (no weight) ───────────────────────────────────
- if (!isKV) {
- float sumOfSquares = 0.0f;
-#pragma unroll
- for (int i = 0; i < kElemsPerLane; i++) {
- sumOfSquares += elements[i] * elements[i];
- }
- sumOfSquares = warpSum<float>(sumOfSquares);
- float const rms_rcp =
- rsqrtf(sumOfSquares / static_cast<float>(kHeadDim) + eps);
-#pragma unroll
- for (int i = 0; i < kElemsPerLane; i++) {
- elements[i] = elements[i] * rms_rcp;
- }
- }
-
- // ── GPT-J RoPE on dims [NOPE_DIM, HEAD_DIM) ─────────────────────────────
- // All math in fp32. cos_sin_cache is loaded as fp32 (its native storage).
- bool const is_rope_lane = dim_base >= kNopeDim;
- if (is_rope_lane) {
- int64_t const pos = position_ids[tokenIdx];
- constexpr int kHalfRope = kRopeDim / 2;
- float const* cos_ptr = cos_sin_cache + pos * kRopeDim;
- float const* sin_ptr = cos_ptr + kHalfRope;
-
- int const rope_local_base = dim_base - kNopeDim;
- int const half_base = rope_local_base >> 1;
-
- // Load phase: 4 vectorized LDGs issue back-to-back.
- float4 const c0 = *reinterpret_cast<float4 const*>(cos_ptr + half_base);
- float4 const c1 = *reinterpret_cast<float4 const*>(cos_ptr + half_base + 4);
- float4 const s0 = *reinterpret_cast<float4 const*>(sin_ptr + half_base);
- float4 const s1 = *reinterpret_cast<float4 const*>(sin_ptr + half_base + 4);
- float const cos_arr[8] = {c0.x, c0.y, c0.z, c0.w, c1.x, c1.y, c1.z, c1.w};
- float const sin_arr[8] = {s0.x, s0.y, s0.z, s0.w, s1.x, s1.y, s1.z, s1.w};
-
-#pragma unroll
- for (int p = 0; p < kElemsPerLane / 2; p++) {
- float const x_even = elements[2 * p];
- float const x_odd = elements[2 * p + 1];
- elements[2 * p] = x_even * cos_arr[p] - x_odd * sin_arr[p];
- elements[2 * p + 1] = x_even * sin_arr[p] + x_odd * cos_arr[p];
- }
- }
-
- // ═══════════════════════════════════════════════════════════════════
- // Q / KV branch dispatch. Restructured as if/else (no early `return`)
- // so every code path lands at the same exit point β€” callers own PDL
- // triggering and per-iteration buffer rotation.
- // ═══════════════════════════════════════════════════════════════════
- if (!isKV) {
- // ── Live-Q: cast back to bf16 and store into the padded q_out. ─────
- uint4 out0, out1;
- typename Converter::packed_hip_type* po0 =
- reinterpret_cast<typename Converter::packed_hip_type*>(&out0);
- typename Converter::packed_hip_type* po1 =
- reinterpret_cast<typename Converter::packed_hip_type*>(&out1);
-#pragma unroll
- for (int i = 0; i < 4; i++) {
- po0[i] =
- Converter::convert(make_float2(elements[2 * i], elements[2 * i + 1]));
- }
-#pragma unroll
- for (int i = 0; i < 4; i++) {
- po1[i] = Converter::convert(
- make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1]));
- }
- scalar_t_in* dst =
- q_out +
- (static_cast<int64_t>(tokenIdx) * kNumHeadsQPadded + slotIdx) *
- kHeadDim +
- dim_base;
- *reinterpret_cast<uint4*>(dst) = out0;
- *reinterpret_cast<uint4*>(dst + 8) = out1;
- } else {
- // ── KV: FP8 quant on NoPE + bf16 store on RoPE + cache insert.
- int64_t const slot_id = slot_mapping[tokenIdx];
- if (slot_id >= 0) {
- int64_t const block_idx = slot_id / cache_block_size;
- int64_t const pos_in_block = slot_id % cache_block_size;
- uint8_t* block_base =
- k_cache + block_idx * static_cast<int64_t>(kv_block_stride);
- uint8_t* token_fp8_ptr = block_base + pos_in_block * kTokenDataBytes;
- uint8_t* token_bf16_ptr = token_fp8_ptr + kNopeDim;
- uint8_t* token_scale_ptr =
- block_base +
- static_cast<int64_t>(cache_block_size) * kTokenDataBytes +
- pos_in_block * kScaleBytesPerToken;
-
-#pragma unroll
- for (int i = 0; i < kElemsPerLane; i++) {
- elements[i] = Converter::convert(Converter::convert(elements[i]));
- }
-
- float local_absmax = 0.0f;
-#pragma unroll
- for (int i = 0; i < kElemsPerLane; i++) {
- local_absmax = fmaxf(local_absmax, fabsf(elements[i]));
- }
- float const absmax = fmaxf(warp4MaxAbs(local_absmax), 1e-4f);
- float const exponent = ceilf(log2f(absmax / kFp8Max));
- float const inv_scale = exp2f(-exponent);
-
- if (!is_rope_lane) {
- uint8_t out_bytes[kElemsPerLane];
-#pragma unroll
- for (int i = 0; i < kElemsPerLane; i++) {
- float scaled = elements[i] * inv_scale;
- scaled = fminf(fmaxf(scaled, -kFp8Max), kFp8Max);
-#ifndef USE_ROCM
- __nv_fp8_storage_t s =
- __nv_cvt_float_to_fp8(scaled, __NV_SATFINITE, __NV_E4M3);
- out_bytes[i] = static_cast<uint8_t>(s);
-#else
- out_bytes[i] = rocm_cvt_float_to_fp8_e4m3(scaled);
-#endif
- }
- *reinterpret_cast<uint4*>(token_fp8_ptr + dim_base) =
- *reinterpret_cast<uint4 const*>(out_bytes);
-
- if ((laneId & 3) == 0) {
- int const q_block_idx = laneId >> 2;
- float encoded = fmaxf(fminf(exponent + 127.0f, 255.0f), 0.0f);
- token_scale_ptr[q_block_idx] = static_cast<uint8_t>(encoded);
- }
- if (laneId == 0) {
- token_scale_ptr[kNumQuantBlocks] = 0;
- }
- } else {
- uint4 out0, out1;
- typename Converter::packed_hip_type* po0 =
- reinterpret_cast<typename Converter::packed_hip_type*>(&out0);
- typename Converter::packed_hip_type* po1 =
- reinterpret_cast<typename Converter::packed_hip_type*>(&out1);
-#pragma unroll
- for (int i = 0; i < 4; i++) {
- po0[i] = Converter::convert(
- make_float2(elements[2 * i], elements[2 * i + 1]));
- }
-#pragma unroll
- for (int i = 0; i < 4; i++) {
- po1[i] = Converter::convert(
- make_float2(elements[8 + 2 * i], elements[8 + 2 * i + 1]));
- }
- int const rope_local_base = dim_base - kNopeDim;
- scalar_t_in* bf16_dst =
- reinterpret_cast<scalar_t_in*>(token_bf16_ptr) + rope_local_base;
- *reinterpret_cast<uint4*>(bf16_dst) = out0;
- *reinterpret_cast<uint4*>(bf16_dst + 8) = out1;
- }
- }
- }
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-// Kernel
-// ────────────────────────────────────────────────────────────────────────────
-//
-// Grid: 1D, gridDim.x = ceil(num_tokens_full * (kNumHeadsQPadded + 1) /
-// warps_per_block) Block: blockDim.x = 256 threads (8 warps per block) Each
-// warp handles one (token, head_slot) pair.
-// slot < num_heads_q β†’ live-Q branch
-// (RMSNorm + RoPE,
-// read q_in β†’ write q_out)
-// num_heads_q <= slot < kNumHeadsQPadded β†’ pad-Q branch
-// (zero-fill q_out)
-// slot == kNumHeadsQPadded β†’ KV branch
-// (RoPE + UE8M0 quant +
-// paged-cache insert)
-//
-// `kNumHeadsQPadded` is a template parameter (compile-time constant) so the
-// divisions in the grid math and the KV-sentinel comparison fold to fast
-// constant operations. The launch wrapper dispatches the runtime value to
-// the matching instantiation.
-//
-// With DP padding, q/kv/position_ids can have more rows than slot_mapping.
-// The live-Q and pad-Q branches cover all `num_tokens_full` rows (downstream
-// attention uses them). The KV branch only inserts the first
-// `num_tokens_insert` tokens (= slot_mapping length) into the paged cache.
-//
-template <typename scalar_t_in, int kNumHeadsQPadded>
-__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel(
- scalar_t_in const* __restrict__ q_in, // [N, num_heads_q, 512]
- scalar_t_in* __restrict__ q_out, // [N, kNumHeadsQPadded, 512]
- scalar_t_in const* __restrict__ kv_in, // [N, 512] bf16
- uint8_t* __restrict__ k_cache, // [num_blocks, block_stride]
- int64_t const* __restrict__ slot_mapping, // [num_tokens_insert] i64
- int64_t const* __restrict__ position_ids, // [N] i64
- float const* __restrict__ cos_sin_cache, // [max_pos, 64] fp32
- float const eps,
- int const num_tokens_full, // = q.size(0) = kv.size(0)
- int const num_tokens_insert, // = slot_mapping.size(0), ≀ num_tokens_full
- int const num_heads_q, // live Q heads (input layout)
- int const cache_block_size, // tokens per paged-cache block
- int const kv_block_stride) { // bytes per paged-cache block
-#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
- // BF16 _typeConvert specialization is unavailable on pre-Ampere. The
- // DeepseekV4 kernel only runs with bf16 inputs in practice, so compile a
- // no-op stub for sm_70/sm_75 to keep multi-arch builds happy.
- if constexpr (std::is_same_v<scalar_t_in, __nv_bfloat16>) {
- return;
- } else {
-#endif
- int const warpsPerBlock = blockDim.x / 32;
- int const warpId = threadIdx.x / 32;
- int const laneId = threadIdx.x % 32;
- int const globalWarpIdx = blockIdx.x * warpsPerBlock + warpId;
-
- constexpr int kTotalSlotsPerToken = kNumHeadsQPadded + 1;
- int const tokenIdx = globalWarpIdx / kTotalSlotsPerToken;
- int const slotIdx = globalWarpIdx % kTotalSlotsPerToken;
- if (tokenIdx >= num_tokens_full) return;
-
- bool const isKV = (slotIdx == kNumHeadsQPadded);
- bool const isPadQ = !isKV && (slotIdx >= num_heads_q);
- // KV branch: skip DP-padded tokens (no slot reserved for them).
- if (isKV && tokenIdx >= num_tokens_insert) return;
-
- // PDL: wait for predecessor kernel (upstream q/kv producer) to signal
- // before touching any global memory. No-op when PDL is not enabled on
- // the launch. The CUDA runtime wrapper emits the griddepcontrol.wait
- // PTX with the required memory clobber internally.
-#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
- cudaGridDependencySynchronize();
-#endif
-
- // Dim range this lane owns within the 512-wide head.
- int const dim_base = laneId * kElemsPerLane; // in [0, 512) step 16
-
- // Load only for live-Q and KV slots; pad-Q skips the read (q_in beyond
- // num_heads_q is out of bounds) and the helper zero-fills its output.
- uint4 v0, v1;
- if (!isPadQ) {
- scalar_t_in const* src_ptr;
- if (isKV) {
- src_ptr = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
- } else {
- int64_t const q_row_offset =
- (static_cast<int64_t>(tokenIdx) * num_heads_q + slotIdx) *
- kHeadDim +
- dim_base;
- src_ptr = q_in + q_row_offset;
- }
- v0 = *reinterpret_cast<uint4 const*>(src_ptr);
- v1 = *reinterpret_cast<uint4 const*>(src_ptr + 8);
- }
-
- processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded>(
- v0, v1, tokenIdx, slotIdx, dim_base, laneId, num_heads_q, eps, q_out,
- k_cache, slot_mapping, position_ids, cos_sin_cache, cache_block_size,
- kv_block_stride);
-
-#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
- cudaTriggerProgrammaticLaunchCompletion();
-#endif
-#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
- }
-#endif
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-// Kernel
-// ────────────────────────────────────────────────────────────────────────────
-//
-// Grid: 1D, gridDim.x = num_tokens_full
-// Block: blockDim.x = 256 threads (8 warps per block) Each
-// warp handles one token, iterating over each head.
-// Q branch (RMSNorm + RoPE, in place) head_slot == num_heads_q
-// KV branch (RoPE + UE8M0 quant + insert)
-//
-template <typename scalar_t_in, int kNumHeadsQPadded>
-__global__ void fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid(
- scalar_t_in const* __restrict__ q_in, scalar_t_in* __restrict__ q_out,
- scalar_t_in const* __restrict__ kv_in, uint8_t* __restrict__ k_cache,
- int64_t const* __restrict__ slot_mapping,
- int64_t const* __restrict__ position_ids,
- float const* __restrict__ cos_sin_cache, float const eps,
- int const num_tokens_full, int const num_tokens_insert,
- int const num_heads_q, int const cache_block_size,
- int const kv_block_stride) {
-#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
- if constexpr (std::is_same_v<scalar_t_in, __nv_bfloat16>) {
- return;
- } else {
-#endif
- int const warpsPerBlock = blockDim.x / 32;
- int const warpId = threadIdx.x / 32;
- int const laneId = threadIdx.x % 32;
-
- int const tokenIdx = blockIdx.x;
- if (tokenIdx >= num_tokens_full) return;
-
-#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
- cudaGridDependencySynchronize();
-#endif
-
- int const dim_base = laneId * kElemsPerLane; // in [0, 512) step 16
- // Slot enumeration: live-Q + pad-Q + (KV if this token has a slot).
- int const slot_end = (tokenIdx >= num_tokens_insert)
- ? kNumHeadsQPadded
- : (kNumHeadsQPadded + 1);
-
- auto load_slot = [&](int s, uint4& va, uint4& vb) {
- // pad-Q slots skip the load β€” q_in beyond num_heads_q is OOB.
- if (s >= num_heads_q && s < kNumHeadsQPadded) return;
- scalar_t_in const* src;
- if (s == kNumHeadsQPadded) {
- src = kv_in + static_cast<int64_t>(tokenIdx) * kHeadDim + dim_base;
- } else {
- src = q_in +
- (static_cast<int64_t>(tokenIdx) * num_heads_q +
- static_cast<int64_t>(s)) *
- kHeadDim +
- dim_base;
- }
- va = *reinterpret_cast<uint4 const*>(src);
- vb = *reinterpret_cast<uint4 const*>(src + 8);
- };
-
- if (warpId < slot_end) {
- int curr_slot = warpId;
- uint4 v0_curr, v1_curr;
- load_slot(curr_slot, v0_curr, v1_curr);
-
- while (curr_slot < slot_end) {
- int const next_slot = curr_slot + warpsPerBlock;
- bool const has_next = (next_slot < slot_end);
-
- // Prefetch src for the next slot
- uint4 v0_next, v1_next;
- if (has_next) {
- load_slot(next_slot, v0_next, v1_next);
- }
-
- processDeepseekV4Slot<scalar_t_in, kNumHeadsQPadded>(
- v0_curr, v1_curr, tokenIdx, curr_slot, dim_base, laneId,
- num_heads_q, eps, q_out, k_cache, slot_mapping, position_ids,
- cos_sin_cache, cache_block_size, kv_block_stride);
-
- // ── Buffer rotation: hand the prefetched LDGs to the next iter.
- v0_curr = v0_next;
- v1_curr = v1_next;
- curr_slot = next_slot;
- } // while
- } // if (warpId < slot_end)
-
-#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
- cudaTriggerProgrammaticLaunchCompletion();
-#endif
-#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
- }
-#endif
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-// Launch wrapper
-// ────────────────────────────────────────────────────────────────────────────
-template <typename scalar_t_in, int kNumHeadsQPadded>
-static void launchFusedDeepseekV4Templated(
- scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
- uint8_t* k_cache, int64_t const* slot_mapping, int64_t const* position_ids,
- float const* cos_sin_cache, float const eps, int const num_tokens_full,
- int const num_tokens_insert, int const num_heads_q,
- int const cache_block_size, int const kv_block_stride,
- cudaStream_t stream) {
- constexpr int kBlockSize = 256;
- constexpr int kWarpsPerBlock = kBlockSize / 32;
- int64_t const total_warps =
- static_cast<int64_t>(num_tokens_full) * (kNumHeadsQPadded + 1);
- int const grid =
- static_cast<int>((total_warps + kWarpsPerBlock - 1) / kWarpsPerBlock);
-
- // PDL: enable programmatic stream serialization whenever the hardware
- // supports it (SM90+). On pre-Hopper GPUs the attribute is unavailable,
- // so leave numAttrs = 0 and launch as a regular kernel.
-#ifndef USE_ROCM
- static int const sm_version = getSMVersion();
- // Host-side guard: the device kernel body is compiled as a no-op for
- // bf16 on pre-Ampere (sm_70/sm_75) because _typeConvert<BFloat16> is
- // unavailable there. Refuse the launch loudly instead of silently
- // skipping the work.
- if (sm_version < 80) return;
- cudaLaunchConfig_t config;
- config.gridDim = dim3(grid);
- config.blockDim = dim3(kBlockSize);
- config.dynamicSmemBytes = 0;
- config.stream = stream;
- cudaLaunchAttribute attrs[1];
- attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
- attrs[0].val.programmaticStreamSerializationAllowed = 1;
- config.attrs = attrs;
- config.numAttrs = (sm_version >= 90) ? 1 : 0;
-
- if (num_tokens_full < NUM_TOKEN_CUTOFF) {
- cudaLaunchKernelEx(
- &config,
- fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<scalar_t_in,
- kNumHeadsQPadded>,
- q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
- eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
- kv_block_stride);
- } else {
- config.gridDim = dim3(num_tokens_full);
- cudaLaunchKernelEx(
- &config,
- fusedDeepseekV4QNormRopeKVRopeQuantInsertKernelReducedGrid<
- scalar_t_in, kNumHeadsQPadded>,
- q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, cos_sin_cache,
- eps, num_tokens_full, num_tokens_insert, num_heads_q, cache_block_size,
- kv_block_stride);
- }
-#else
- // ROCm: use standard kernel launch syntax (no PDL/stream serialization)
- // clang-format off
- fusedDeepseekV4QNormRopeKVRopeQuantInsertKernel<scalar_t_in, kNumHeadsQPadded>
- <<<grid, kBlockSize, 0, stream>>>(
- q_in, q_out, kv_in, k_cache, slot_mapping, position_ids,
- cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q,
- cache_block_size, kv_block_stride);
-#endif
-}
-
-// Runtime dispatch into one of the precompiled `kNumHeadsQPadded`
-// instantiations. Supported padded head counts: 8, 16, 32, 64, 128.
-template <typename scalar_t_in>
-void launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
- scalar_t_in const* q_in, scalar_t_in* q_out, scalar_t_in const* kv_in,
- uint8_t* k_cache, int64_t const* slot_mapping,
- int64_t const* position_ids, float const* cos_sin_cache, float const eps,
- int const num_tokens_full, int const num_tokens_insert,
- int const num_heads_q, int const num_heads_q_padded,
- int const cache_block_size, int const kv_block_stride,
- cudaStream_t stream) {
-#define DISPATCH(N) \
- case N: \
- launchFusedDeepseekV4Templated<scalar_t_in, N>( \
- q_in, q_out, kv_in, k_cache, slot_mapping, position_ids, \
- cos_sin_cache, eps, num_tokens_full, num_tokens_insert, num_heads_q, \
- cache_block_size, kv_block_stride, stream); \
- return;
-
- switch (num_heads_q_padded) {
- DISPATCH(8)
- DISPATCH(16)
- DISPATCH(32)
- DISPATCH(64)
- DISPATCH(128)
- default: return;
- }
-#undef DISPATCH
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-// FlashInfer full-cache kernel
-// ────────────────────────────────────────────────────────────────────────────
-
-} // namespace deepseek_v4_fused_ops
-} // namespace vllm
-
-extern "C" int dsv4_vllm_qnorm_rope_kv_insert_batch(
- const __nv_bfloat16 *q_in, __nv_bfloat16 *q_out,
- const __nv_bfloat16 *kv_in, uint8_t *kv_cache,
- const int64_t *slot_mapping, const int64_t *position_ids,
- const float *cos_sin_cache, float eps, int tokens, int insert_tokens,
- int heads, int padded_heads,
- int cache_block_size, int kv_block_stride, cudaStream_t stream) {
- if (!q_in || !q_out || !kv_in || !kv_cache || !slot_mapping ||
- !position_ids || !cos_sin_cache || tokens < 1 || insert_tokens < 0 ||
- insert_tokens > tokens) return 0;
- vllm::deepseek_v4_fused_ops::launchFusedDeepseekV4QNormRopeKVRopeQuantInsert(
- q_in, q_out, kv_in, kv_cache, slot_mapping, position_ids,
- cos_sin_cache, eps, tokens, insert_tokens, heads, padded_heads, cache_block_size,
- kv_block_stride, stream);
- return cudaGetLastError() == cudaSuccess;
-}
-
-extern "C" int dsv4_vllm_qnorm_rope_kv_insert(
- const __nv_bfloat16 *q_in, __nv_bfloat16 *q_out,
- const __nv_bfloat16 *kv_in, uint8_t *kv_cache,
- const int64_t *slot_mapping, const int64_t *position_ids,
- const float *cos_sin_cache, float eps, int heads,
- int cache_block_size, int kv_block_stride, cudaStream_t stream) {
- return dsv4_vllm_qnorm_rope_kv_insert_batch(
- q_in, q_out, kv_in, kv_cache, slot_mapping, position_ids, cos_sin_cache,
- eps, 1, 1, heads, heads, cache_block_size, kv_block_stride, stream);
-}
-
-__global__ void packCompressedKV(const float *kv,uint8_t *cache,const int *state,int ratio,
- int cache_block_size,int kv_block_stride) {
- int pos=state[1];if(!ratio||(pos+1)%ratio)return;int slot=(pos+1)/ratio-1;kv+=(long long)slot*512;
- int tile=blockIdx.x,lane=threadIdx.x,block=slot/cache_block_size,row=slot%cache_block_size;
- uint8_t *base=cache+(long long)block*kv_block_stride;
- if(tile<7){
- __shared__ float mx[64];float v=__bfloat162float(__float2bfloat16(kv[tile*64+lane]));
- mx[lane]=fabsf(v);__syncthreads();
- for(int n=32;n;n>>=1){if(lane<n)mx[lane]=fmaxf(mx[lane],mx[lane+n]);__syncthreads();}
- float exponent=ceilf(log2f(fmaxf(mx[0],1e-4f)/448.f)),scaled=fminf(fmaxf(v*exp2f(-exponent),-448.f),448.f);
- base[(long long)row*576+tile*64+lane]=(uint8_t)__nv_cvt_float_to_fp8(scaled,__NV_SATFINITE,__NV_E4M3);
- if(!lane)base[(long long)cache_block_size*576+(long long)row*8+tile]=(uint8_t)fminf(fmaxf(exponent+127.f,0.f),255.f);
- }else{
- __nv_bfloat16 *rope=(__nv_bfloat16*)(base+(long long)row*576+448);
- rope[lane]=__float2bfloat16(kv[448+lane]);
- if(!lane)base[(long long)cache_block_size*576+(long long)row*8+7]=0;
- }
-}
-
-extern "C" int dsv4_vllm_pack_compressed(const float *kv,uint8_t *cache,const int *decode_state,
- int ratio,
- int cache_block_size,int kv_block_stride,
- cudaStream_t stream){
- if(!kv||!cache||!decode_state||ratio<1||cache_block_size<1)return 0;
- packCompressedKV<<<8,64,0,stream>>>(kv,cache,decode_state,ratio,cache_block_size,kv_block_stride);
- return cudaGetLastError()==cudaSuccess;
-}
diff --git a/c/backend_cuda_dsv4_qkv_vllm.h b/c/backend_cuda_dsv4_qkv_vllm.h
deleted file mode 100644
index dd5d7281..00000000
--- a/c/backend_cuda_dsv4_qkv_vllm.h
+++ /dev/null
@@ -1,26 +0,0 @@
-#ifndef COLIBRI_BACKEND_CUDA_DSV4_QKV_VLLM_H
-#define COLIBRI_BACKEND_CUDA_DSV4_QKV_VLLM_H
-#include <cuda_runtime.h>
-#include <stdint.h>
-#ifdef __cplusplus
-extern "C" {
-#endif
-int dsv4_vllm_qnorm_rope_kv_insert(const void *q_in,void *q_out,const void *kv_in,
- uint8_t *kv_cache,const int64_t *slot_mapping,
- const int64_t *position_ids,const float *cos_sin_cache,
- float eps,int heads,int cache_block_size,
- int kv_block_stride,cudaStream_t stream);
-int dsv4_vllm_qnorm_rope_kv_insert_batch(const void *q_in,void *q_out,const void *kv_in,
- uint8_t *kv_cache,const int64_t *slot_mapping,
- const int64_t *position_ids,const float *cos_sin_cache,
- float eps,int tokens,int insert_tokens,int heads,
- int padded_heads,int cache_block_size,
- int kv_block_stride,cudaStream_t stream);
-int dsv4_vllm_pack_compressed(const float *kv,uint8_t *cache,const int *decode_state,
- int ratio,
- int cache_block_size,int kv_block_stride,
- cudaStream_t stream);
-#ifdef __cplusplus
-}
-#endif
-#endif
diff --git a/c/tests/bench_dsv4_flashinfer.cu b/c/tests/bench_dsv4_flashinfer.cu
deleted file mode 100644
index 4b9ff790..00000000
--- a/c/tests/bench_dsv4_flashinfer.cu
+++ /dev/null
@@ -1,81 +0,0 @@
-#include <cuda_runtime.h>
-#include <flashinfer/gemm/group_gemm_mxfp4_groupwise_sm120.cuh>
-#include <cmath>
-#include <cstdlib>
-#include <cstdio>
-#include <vector>
-
-namespace flashinfer::group_gemm {
-INSTANTIATE_GROUP_GEMM_MXFP4_GROUPWISE_SM120(
- 128, 32, 128, true, cutlass::float_e4m3_t, cutlass::float_e2m1_t,
- cutlass::float_ue8m0_t, cutlass::float_ue8m0_t, cutlass::bfloat16_t,
- fp8, fp4, ue8m0, ue8m0, bf16)
-}
-
-using namespace flashinfer::group_gemm;
-
-static void check(cudaError_t e, const char *where) {
- if (e != cudaSuccess) {
- std::fprintf(stderr, "%s: %s\n", where, cudaGetErrorString(e));
- std::exit(1);
- }
-}
-
-int main(int argc, char **argv) {
- int groups = argc > 1 ? std::atoi(argv[1]) : 6;
- int n = argc > 2 ? std::atoi(argv[2]) : 2048;
- int k = argc > 3 ? std::atoi(argv[3]) : 4096;
- constexpr int rows = 4;
- int total_rows = groups * rows;
- constexpr size_t int_workspace_bytes = 8 << 20;
- constexpr size_t float_workspace_bytes = 64 << 20;
- cutlass::float_e4m3_t *a;
- cutlass::float_e2m1_t *b;
- cutlass::float_ue8m0_t *as, *bs;
- cutlass::bfloat16_t *out;
- int *indptr;
- void *int_workspace, *float_workspace;
- check(cudaMalloc(&a, (size_t)total_rows * k), "allocate activation");
- check(cudaMalloc(&b, (size_t)groups * n * k / 2), "allocate weight");
- check(cudaMalloc(&as, (size_t)groups * 128 * (k / 32)), "allocate activation scales");
- check(cudaMalloc(&bs, (size_t)groups * n * (k / 32)), "allocate weight scales");
- check(cudaMalloc(&out, (size_t)total_rows * n * sizeof(*out)), "allocate output");
- check(cudaMalloc(&indptr, (groups + 1) * sizeof(*indptr)), "allocate indptr");
- check(cudaMalloc(&int_workspace, int_workspace_bytes), "allocate integer workspace");
- check(cudaMalloc(&float_workspace, float_workspace_bytes), "allocate CUTLASS workspace");
- check(cudaMemset(a, 0x38, (size_t)total_rows * k), "initialize unit FP8 activation");
- check(cudaMemset(b, 0x22, (size_t)groups * n * k / 2), "initialize unit FP4 weight");
- check(cudaMemset(as, 127, (size_t)groups * 128 * (k / 32)), "initialize activation scales");
- check(cudaMemset(bs, 127, (size_t)groups * n * (k / 32)), "initialize weight scales");
- std::vector<int> host_indptr(groups + 1);
- for (int i = 0; i <= groups; ++i) host_indptr[i] = i * rows;
- check(cudaMemcpy(indptr, host_indptr.data(), (groups + 1) * sizeof(*indptr), cudaMemcpyHostToDevice),
- "upload indptr");
- auto run = [&] {
- check(CutlassMXFP4GroupwiseScaledGroupGEMMSM120<
- 128, 32, 128, true, cutlass::float_e4m3_t, cutlass::float_e2m1_t,
- cutlass::float_ue8m0_t, cutlass::float_ue8m0_t, cutlass::bfloat16_t>(
- int_workspace, int_workspace_bytes, float_workspace, float_workspace_bytes,
- a, b, as, bs, out, indptr, n, k, groups, nullptr, 0),
- "launch grouped MXFP4 GEMM");
- };
- for (int i = 0; i < 10; ++i) run();
- check(cudaDeviceSynchronize(), "warmup");
- cutlass::bfloat16_t first;
- check(cudaMemcpy(&first, out, sizeof(first), cudaMemcpyDeviceToHost), "download correctness sample");
- if (std::abs(float(first) - k) > k / 100.f) {
- std::fprintf(stderr, "wrong MXFP4 result: got %.3f expected %d\n", float(first), k);
- return 2;
- }
- cudaEvent_t begin, end;
- check(cudaEventCreate(&begin), "create begin event");
- check(cudaEventCreate(&end), "create end event");
- check(cudaEventRecord(begin), "record begin");
- for (int i = 0; i < 100; ++i) run();
- check(cudaEventRecord(end), "record end");
- check(cudaEventSynchronize(end), "benchmark");
- float elapsed_ms = 0;
- check(cudaEventElapsedTime(&elapsed_ms, begin, end), "elapsed time");
- std::printf("flashinfer_sm120_mxfp4 groups=%d n=%d k=%d %.3f us/call\n",
- groups, n, k, elapsed_ms * 10.f);
-}