FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(gguf): serve GGUF checkpoints on vendored ggml-cuda kernels (#690)
Xiaoze Fane5d3bd3fafb7
diff --git a/benchmarks/bench_load_weight_generic.py b/benchmarks/bench_load_weight_generic.pyindex 4e4c5fe0..89ddb307 100644--- a/benchmarks/bench_load_weight_generic.py+++ b/benchmarks/bench_load_weight_generic.py@@ -179,7 +179,7 @@ def _bench_load(mode: str, model_path: str, *, parallel: bool, workers: int, chut = time.perf_counter()try:banks = load_expert_banks(model_path, mc, device=torch.device("cuda", torch.cuda.current_device()),- dtype=torch.bfloat16, parallel=parallel,+ parallel=parallel,workers=workers, chunk=chunk)except NotImplementedError as e:s.stop()diff --git a/docs/cli.md b/docs/cli.mdindex 44b0e30a..4918a317 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -189,7 +189,7 @@ ft checkpoint --model <hf_dir> --out <ftw_dir> [--dtype bfloat16] [--moe-backendConverts an HF safetensors checkpoint to FTW, FreeToken's self-containedfast-load format; point `ft serve --model` at the output dir. MoE experts arealways packed into expert banks, which serve the fused and the offload-strategies alike (GGUF and DeepSeek-V4 experts serve offload only);+strategies alike (DeepSeek-V4 experts serve offload only);`--moe-backend` is kept for compatibility. See the FTW caveats in[models.md](models.md#notes); FTW files from older builds can be repaired with[scripts/ftw_hotfix.py](ftw-hotfix.md) instead of reconverting.diff --git a/docs/models.md b/docs/models.mdindex cdadc2dc..d3f26a08 100644--- a/docs/models.md+++ b/docs/models.md@@ -42,7 +42,7 @@ These families accept image input by default; pass `--text-model-only` to skip t`ft serve --moe-strategy {auto,fused,offload,cpu,hybrid}` (`--moe-backend` is the deprecated old spelling):- **fused** — experts resident on GPU (needs the VRAM). Serves every expert- format except GGUF and DeepSeek-V4's.+ format except DeepSeek-V4's.- **offload** — experts live in host RAM, an LRU cache of expert slots on GPU;misses stream over PCIe.- **cpu** — misses are computed on the CPU instead of fetched.@@ -56,7 +56,7 @@ These families accept image input by default; pass `--text-model-only` to skip tThe GPU and the CPU share one DRAM, so offload only copies experts between twonames for the same memory. On these GPUs `auto` resolves every MoE model to-`fused`: the default is the resident path only. GGUF and DeepSeek-V4 experts+`fused`: the default is the resident path only. DeepSeek-V4 expertshave no resident path and stay on `offload`. Offload-only flags(`--moe-cpu-layers`, `--moe-cache-size` / `-rate` / `-auto`,`--moe-prefill-hit-d2d`, `--disable-moe-prefill-overlap`) are ignored with adiff --git a/pyproject.toml b/pyproject.tomlindex b308045f..ab1dfd07 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -40,7 +40,6 @@ dependencies = ["fastapi>=0.115,<1",# slot_cache: the device-side LRU admission kernel behind the MoE expert cache."flashlib==0.4.0", # pin it- "gguf>=0.19,<1","huggingface_hub>=1.5,<2","msgpack>=1.1,<2","modelscope>=1.37,<2",@@ -113,7 +112,7 @@ version = {attr = "freetoken.version.__version__"}where = ["python"][tool.setuptools.package-data]-"*" = ["csrc/**/*", "moe/configs/**/*.json"]+"*" = ["csrc/**/*", "moe/configs/**/*.json", "third_party/*/LICENSE", "third_party/*/VERSION", "third_party/*/freetoken.patch"][tool.pytest.ini_options]testpaths = ["tests"]diff --git a/python/freetoken/checkpoint/convert.py b/python/freetoken/checkpoint/convert.pyindex 152f3871..054fade4 100644--- a/python/freetoken/checkpoint/convert.py+++ b/python/freetoken/checkpoint/convert.py@@ -247,7 +247,7 @@ def convert_checkpoint(names = legacy_bank_names(legacy_format_for(method.kind, method.kernel.name)) if method is not None else {}sink = _ConvertSink(writer, names=names)banks = load_expert_banks(- model_path, mc, method=method, device=dev, dtype=dtype, layer_sink=sink+ model_path, mc, method=method, device=dev, layer_sink=sink)quant_format = banks.quant_formatif banks.streamed:diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.pyindex ae2f1fd6..a6c47dd0 100644--- a/python/freetoken/engine/config.py+++ b/python/freetoken/engine/config.py@@ -141,7 +141,7 @@ class EngineConfig:hf_config = copy.copy(self.hf_config)built = {e.config_key for e in self.active_encoders}for key in set(ENCODER_SECTIONS) | {e.config_key for e in self.model_spec.encoders}:- if key not in built:+ if key not in built and getattr(hf_config, key, None) is not None:setattr(hf_config, key, None)spec = self.model_specquant = checkpoint_quant_config(self.model_path, hf_config, spec)diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 2757b10d..81f42dba 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -15,7 +15,7 @@ from freetoken.distributed import destroy_distributed, enable_pynccl_distributedfrom freetoken.gpu_select import gpu_identityfrom freetoken.checkpoint.ftw import ftw_tensor_names, is_ftw_checkpointfrom freetoken.layers import OffloadMoELayer, iter_moe_layers, set_rope_device-from freetoken.layers.quantization import LayerKind, QuantBackend, finalize_quant, set_quant_backend+from freetoken.layers.quantization import LayerKind, QuantBackend, QuantKind, finalize_quant, set_quant_backendfrom freetoken.moe.offload_cache import iter_offload_moe_layersfrom freetoken.mm.config import ENCODER_SECTIONSfrom freetoken.models import create_model, load_weight@@ -304,7 +304,7 @@ def _make_dummy_weight_state_dict(# MXFP8 e8m0 exponent codes: 127 encodes scale 1.0; zeros would collapse# every scale to 2^-127 and zero the model. Scoped BY NAME: other uint8# buffers are packed payloads whose bytes mean something else entirely- # (GGUF qweight blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they+ # (GGUF blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they# keep the benign all-zeros fill below.state_dict[key] = torch.full(param.shape, 127, dtype=param.dtype, device=device)else:@@ -603,7 +603,6 @@ class Engine:config.model_config,method=layers[0].quant_method,device=self.device,- dtype=self.dtype,dummy=config.use_dummy_weight,parallel={"serial": False, "parallel": True}.get(config.expert_load, None),resident=True,@@ -735,10 +734,8 @@ class Engine:config.model_config,method=method,device=self.device,- dtype=self.dtype,dummy=config.use_dummy_weight,parallel=expert_parallel,- decode_target=("cpu" if decode_target in ("cpu", "hybrid") else "gpu"),layer_residency=requested_residency,)except PinFailed as exc:@@ -808,7 +805,8 @@ class Engine:f"expert banks were packed for {banks.kind} / {banks.kernel} but the model "f"binds {method.kind} / {method.kernel.name}; reconvert or select that kernel")- layout = method.layout()+ # GGUF layers keep their own ggml types, so the cache reads each layer's layout off its banks+ layout = None if method.kind is QuantKind.GGUF else method.layout()max_slots = method.slot_limit()cache = OffloadMoeCache(# Models with leading dense layers (GLM-4) only have experts on the MoE@@ -830,7 +828,7 @@ class Engine:)# before set_bank_sources: the residency validation and the copy plan's skip of non-pinned layers key on the CPU-layer setcache.cpu_layer_ids = cpu_layer_ids- cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=banks.layer_tags)+ cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency, layer_tags=banks.layer_tags, row_tail_bytes=banks.row_tail_bytes)cache.set_alphas(banks.gate_up_alpha, banks.down_alpha)if decode_target == "hybrid":self._resolve_hybrid_fetch(config, cache)@@ -839,7 +837,6 @@ class Engine:cache.collect_stats = config.moe_collect_statslayers = attach_offload_moe_cache(self.model, cache)assert len(layers) == config.model_config.num_moe_layers- cache.zero_inactive_route_slots = cache.decode_target == "hybrid" and not all(layer.skips_inactive_slots for layer in layers)if cache.decode_target in ("cpu", "hybrid"):self._init_cpu_moe_executor(config, cache, layers)self.ctx.moe_offload_cache = cache@@ -907,6 +904,9 @@ class Engine:# round a batch up to the largest captured size; cover both.max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)method = sample.quant_method+ formats = {layer.quant_method.cpu_format for layer in layers}+ if len(formats) > 1:+ raise NotImplementedError(f"CPU MoE backend needs one expert format across the MoE layers, got {sorted(map(str, formats))}")executor = CpuMoeExecutor(cache,top_k=sample.top_k,@@ -917,9 +917,8 @@ class Engine:device=self.device,swiglu_alpha=float(sample.alpha),swiglu_limit=sample.limit,- # FIXME: the None branch serves GGUF q4_0 banks, which have no quant method yet; drop it once GGUF joins the quant path- fmt=method.cpu_format if method is not None else None,- act_block=method.cfg.act_block if method is not None else None,+ fmt=method.cpu_format,+ act_block=method.cfg.act_block,)cache.set_cpu_executor(executor)self.cpu_moe_executor = executor@@ -1255,11 +1254,11 @@ def _is_unified_memory_gpu(index: "int | None" = None) -> bool:return False-_RESIDENT_EXPERT_QUANTS = ("none", "fp8_block", "nvfp4")+_RESIDENT_EXPERT_QUANTS = ("none", "fp8_block", "nvfp4", "gguf")def _fused_resident_ok(model_config) -> bool:- """Whether the resident ('fused') MoE path can hold this model's experts; GGUF q4_0 and DeepSeek-V4 experts stay offload."""+ """Whether the resident ('fused') MoE path can hold this model's experts; DeepSeek-V4 experts stay offload."""if getattr(model_config, "expert_quant", "none") not in _RESIDENT_EXPERT_QUANTS:return Falsereturn getattr(model_config, "moe_weight_format", None) in (None, "bf16", "mxfp4")@@ -1752,7 +1751,7 @@ def _adjust_config(config: EngineConfig):if is_moe and config.moe_strategy == "auto":# A MoE model always defaults to the offload family: experts stream from pinned host# banks into an auto-sized GPU slot cache, which is the only default that serves a model- # bigger than the GPU. The resident 'fused' path (every expert format but GGUF q4_0 and+ # bigger than the GPU. The resident 'fused' path (every expert format but# DeepSeek-V4's) is still reachable, but only when asked for explicitly# -- auto never picks it, because nothing here knows whether the experts would fit in# HBM and a wrong guess is a weight-load OOM rather than a slower-but-working run.@@ -1790,7 +1789,7 @@ def _adjust_config(config: EngineConfig):# offload -- same auto-sized GPU slot cache (_resolve_auto_moe_cache_size), plus a# host-RAM CPU executor -- so this never raises the OOM risk; with no profile (or one# from different hardware) it stays offload. offload remains the always-safe fallback.- # Key the lookup on the real expert format: mxfp4/q4_0 live in moe_weight_format when+ # Key the lookup on the real expert format: mxfp4 lives in moe_weight_format when# expert_quant is "none", and "none" with no weight format means plain bf16 experts.moe_wfmt = getattr(model_config, "moe_weight_format", None)bench_fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16")diff --git a/python/freetoken/kernel/aot.py b/python/freetoken/kernel/aot.pyindex bbc2d2c6..76250af7 100644--- a/python/freetoken/kernel/aot.py+++ b/python/freetoken/kernel/aot.py@@ -143,6 +143,15 @@ def _radix_spec() -> KernelSpec:return KernelSpec(name=_make_name("radix"), build=build)+def _ggml_spec() -> KernelSpec:+ def build(build_directory: pathlib.Path) -> object:+ from .ggml import load_module++ return load_module(str(build_directory))++ return KernelSpec(name=_make_name("ggml"), build=build)++def default_kernel_specs() -> tuple[KernelSpec, ...]:specs: list[KernelSpec] = []specs.extend(_store_spec(element_size) for element_size in DEFAULT_STORE_ELEMENT_SIZES)@@ -158,6 +167,7 @@ def default_kernel_specs() -> tuple[KernelSpec, ...]:specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=64))specs.append(_batch_memcpy_spec())specs.append(_radix_spec())+ specs.append(_ggml_spec())names = [spec.name for spec in specs]duplicates = {name for name in names if names.count(name) > 1}diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex ecd1f850..308c2d68 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -16,7 +16,7 @@ JIT, which needs nvcc):DSV4 writes its MLA latent via torch scatter and contributes nothing.- index: ``element_size = hidden_size * 2`` (bf16 embedding row) paired withthe runtime ``num_splits_for`` rule (layers/embedding.py -> kernel/index.py).- DSV4 (plain nn.Embedding) and GGUF embeddings (GGUFEmbedding) bypass it.+ DSV4 (plain nn.Embedding) and GGUF embeddings (ggml get_rows) bypass it.The whole table targets the shipped serving configuration: TP=1 (TP>1 shardskv heads, shrinking the store row) and a 2-byte compute dtype (``--dtype@@ -91,9 +91,6 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int"down": H * I,"down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2,}- if fmt == "q4_0":- # gemma4/gguf.py _q4_0_expert_specs: GGML Q4_0 rows, 32 elems -> 18 bytes- return {"gate_up": 2 * I * (H // 32 * 18), "down": H * (I // 32 * 18)}if fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"):# models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp16# per-row globals; marlin/b12x repacks are byte-identical with the globals@@ -205,6 +202,8 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (top_k=10,moe_intermediate_size=640,expert_formats=(*_NVFP4_FORMATS, "fp8_block"),+ # GGUF expert rows differ per layer and reach the fused copy as runtime sizes+ arch_aliases=("Qwen4ExpGGUFForCausalLM",),),AotModel(name="google/gemma-4-26B-A4B-it",@@ -215,9 +214,8 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (kv_groups=((2, 512), (8, 256)),top_k=8,moe_intermediate_size=704,- # q4_0 = the Q4_0 GGUF release (Gemma4GGUFForCausalLM); its GGUFEmbedding- # path skips indexing(), but the bf16 safetensors path needs the variant.- expert_formats=("bf16", "q4_0"),+ expert_formats=("bf16",),+ # GGUF expert rows differ per layer and reach the fused copy as runtime sizesarch_aliases=("Gemma4ForCausalLM", "Gemma4GGUFForCausalLM"),),AotModel(diff --git a/python/freetoken/kernel/csrc/ggml/LICENSE b/python/freetoken/kernel/csrc/ggml/LICENSEnew file mode 100644index 00000000..e7dca554--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/LICENSE@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2023-2026 The ggml authors++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/python/freetoken/kernel/csrc/ggml/VERSION b/python/freetoken/kernel/csrc/ggml/VERSIONnew file mode 100644index 00000000..6eabb0c1--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/VERSION@@ -0,0 +1,13 @@+llama.cpp 8c2134a8755d8a9792af513dab20282affe903e4 (2026-10-14)++Only the ggml files that the GGUF ops (freetoken/ggml_ops.cu, built by kernel/ggml.py) compile or include:+ggml/include -> include, ggml/src -> src, ggml/src/ggml-cuda -> ggml-cuda.++Local changes, kept in freetoken.patch (already applied):+- quantize.cu/.cuh: the q8_1 quantize kernels read fp32, bf16 or fp16 activations.+- mmq.cu/.cuh: MMQ takes bf16 or fp16 activations, except on the native fp4 path. Its kernels are built for fp32 and bf16 results (dst_t); a bf16 result that needs the stream-k fixup is computed in fp32 first, and so is an fp16 result unless GGML_MMQ_F16_DST builds fp16 kernels too.+- mmvq.cu/.cuh: MMVQ takes bf16 or fp16 activations, can write bf16 or fp16 results, and can reuse activations already quantized to q8_1 (src1_q8_1_pre).+- mmq.cu/.cuh, mmid.cu/.cuh: MoE MMQ runs a flat grid over only the (expert, column tile) pairs that hold tokens, listed by mm_ids_tile_map; ncols_max > 0 keeps the regular grid.+- common.cuh: the native fp4 MMA (BLACKWELL_MMA_AVAILABLE, blackwell_mma_available) is off unless GGML_CUDA_FP4_MMA is defined; it needs an sm_120a/sm_120f build.++To update: copy the same files from a newer llama.cpp, apply freetoken.patch, and update this file.diff --git a/python/freetoken/kernel/csrc/ggml/freetoken.patch b/python/freetoken/kernel/csrc/ggml/freetoken.patchnew file mode 100644index 00000000..3dc715fe--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/freetoken.patch@@ -0,0 +1,994 @@+--- a/ggml/src/ggml-cuda/common.cuh++++ b/ggml/src/ggml-cuda/common.cuh+@@ -293,7 +293,8 @@+ #define AMPERE_MMA_AVAILABLE+ #endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE++-#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN++// FreeToken: the native fp4 MMA needs an sm_120a/sm_120f build, so it is off unless GGML_CUDA_FP4_MMA is defined; FP4 weights then take Q8_1 activations++#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN && defined(GGML_CUDA_FP4_MMA)+ # define BLACKWELL_MMA_AVAILABLE+ #endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL++@@ -374,8 +375,13 @@+ }++ static bool blackwell_mma_available(const int cc) {++#ifdef GGML_CUDA_FP4_MMA+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL &&+ ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;++#else++ GGML_UNUSED(cc);++ return false;++#endif // GGML_CUDA_FP4_MMA+ }++ // Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes.+--- a/ggml/src/ggml-cuda/mmid.cu++++ b/ggml/src/ggml-cuda/mmid.cu+@@ -177,3 +177,45 @@+ break;+ }+ }++++// Lists the (expert, column tile) pairs that hold columns for a flat MoE grid, in expert order.++// The other ntiles grid columns get tile jt_none of expert 0, which starts past its columns, so mul_mat_q skips them.++template <int nthreads>++__launch_bounds__(nthreads, 1)++static __global__ void mm_ids_tile_map(++ const int32_t * __restrict__ expert_bounds, int2 * __restrict__ tile_map, const int n_experts, const int J, const int ntiles, const int jt_none) {++ __shared__ int scan[nthreads];++++ int start = 0; // First tile of the current chunk of experts.++ for (int e0 = 0; e0 < n_experts; e0 += nthreads) {++ const int e = e0 + threadIdx.x;++ const int n = e < n_experts ? (expert_bounds[e + 1] - expert_bounds[e] + J - 1) / J : 0;++++ scan[threadIdx.x] = n;++ __syncthreads();++#pragma unroll++ for (int offset = 1; offset < nthreads; offset *= 2) {++ const int add = threadIdx.x >= offset ? scan[threadIdx.x - offset] : 0;++ __syncthreads();++ scan[threadIdx.x] += add;++ __syncthreads();++ }++++ const int first = start + scan[threadIdx.x] - n;++ for (int jt = 0; jt < n; ++jt) {++ tile_map[first + jt] = make_int2(e, jt);++ }++ start += scan[nthreads - 1];++ __syncthreads();++ }++++ for (int t = start + threadIdx.x; t < ntiles; t += nthreads) {++ tile_map[t] = make_int2(0, jt_none);++ }++}++++void ggml_cuda_launch_mm_ids_tile_map(++ const int32_t * expert_bounds, int2 * tile_map, const int n_experts, const int J, const int ntiles, const int jt_none, cudaStream_t stream) {++ constexpr int nthreads = 512;++ mm_ids_tile_map<nthreads><<<1, nthreads, 0, stream>>>(expert_bounds, tile_map, n_experts, J, ntiles, jt_none);++}+--- a/ggml/src/ggml-cuda/mmid.cuh++++ b/ggml/src/ggml-cuda/mmid.cuh+@@ -3,3 +3,6 @@+ void ggml_cuda_launch_mm_ids_helper(+ const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds,+ int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream);++++void ggml_cuda_launch_mm_ids_tile_map(++ const int32_t * expert_bounds, int2 * tile_map, int n_experts, int J, int ntiles, int jt_none, cudaStream_t stream);+--- a/ggml/src/ggml-cuda/mmq.cu++++ b/ggml/src/ggml-cuda/mmq.cu+@@ -132,9 +132,11 @@+ }++ void ggml_cuda_mul_mat_q(+- ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {+- GGML_ASSERT( src1->type == GGML_TYPE_F32);+- GGML_ASSERT( dst->type == GGML_TYPE_F32);++ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,++ int64_t ncols_max) {++ // FreeToken: bf16 and fp16 activations are quantized directly, except for the native fp4 paths; dst is fp32, bf16 or fp16++ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);++ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.++ GGML_TENSOR_BINARY_OP_LOCALS;+@@ -156,7 +158,9 @@++ const char * src0_d = (const char *) src0->data;+ const float * src1_d = (const float *) src1->data;+- float * dst_d = (float *) dst->data;++ const mmq_dst dst_d = {dst->data, dst->type};++ const nv_bfloat16 * src1_bf16 = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1->data : nullptr;++ const half * src1_f16 = src1->type == GGML_TYPE_F16 ? (const half *) src1->data : nullptr;++ // If src0 is a temporary compute buffer, clear any potential padding.+ if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {+@@ -183,6 +187,7 @@+ const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc);++ const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;++ GGML_ASSERT(!use_native_fp4 || src1->type == GGML_TYPE_F32);+ const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);+ const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ;++@@ -192,7 +197,8 @@+ int64_t ncols_opt = ne11;+ if (ids) {+ const int64_t n_expert_used = ids->ne[0];+- ncols_opt = ne12;++ // a flat grid runs only the tiles that hold tokens, so it sizes them for the average expert++ ncols_opt = ncols_max > 0 ? ncols_max : (ne12*n_expert_used + ne02 - 1) / ne02;++ // Each expert only sees ne12*n_expert_used/ne02 tokens on average.+ // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.+@@ -249,6 +255,12 @@+ quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded,+ ne11, ne12, ne13, stream);+++ } else if (src1_bf16) {++ quantize_mmq_q8_1_cuda(src1_bf16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,++ ne11, ne12, ne13, stream);++ } else if (src1_f16) {++ quantize_mmq_q8_1_cuda(src1_f16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,++ ne11, ne12, ne13, stream);+ } else {+ quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,+ ne11, ne12, ne13, stream);+@@ -299,6 +311,17 @@+ CUDA_CHECK(cudaGetLastError());+ }+++ // Every expert's column tiles fit in ntx_flat grid columns, so a flat grid needs no host sync to size it.++ ggml_cuda_pool_alloc<int2> tile_map(ctx.pool());++ int ntx_flat = 0;++ if (ncols_max <= 0) {++ const int jt_none = (ne_get_rows + J_best - 1) / J_best;++ ntx_flat = jt_none + std::min<int64_t>(ne02, ne_get_rows);++ tile_map.alloc(ntx_flat);++ ggml_cuda_launch_mm_ids_tile_map(expert_bounds.get(), tile_map.get(), ne02, J_best, ntx_flat, jt_none, stream);++ CUDA_CHECK(cudaGetLastError());++ }+++ const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);+ ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());+@@ -325,9 +348,21 @@+ quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13,+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);+ }++ } else if (dedup_bcast && src1_bf16) {++ quantize_scatter_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,++ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);++ } else if (dedup_bcast && src1_f16) {++ quantize_scatter_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,++ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);+ } else if (dedup_bcast) {+ quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);++ } else if (src1_bf16) {++ quantize_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,++ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);++ } else if (src1_f16) {++ quantize_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,++ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);+ } else {+ quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);+@@ -347,7 +382,7 @@+ ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,+ ne02, ne02, s02, s12, s2,+ ne03, ne13, s03, s13, s3,+- ne12, J_best};++ ncols_max > 0 ? ncols_max : ne12, J_best, tile_map.ptr, ntx_flat};++ ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);+ }+--- a/ggml/src/ggml-cuda/mmq.cuh++++ b/ggml/src/ggml-cuda/mmq.cuh+@@ -10,6 +10,17 @@+ #define MMQ_ITER_K_FP4 512+ #define MMQ_NWARPS 8+++// FreeToken: MMQ writes fp32, bf16 or fp16 results (dst_t); the sums stay fp32 until the store++template <typename T> static __device__ __forceinline__ T mmq_cast(float v);++template <> __device__ __forceinline__ float mmq_cast<float>(float v) { return v; }++template <> __device__ __forceinline__ nv_bfloat16 mmq_cast<nv_bfloat16>(float v) { return __float2bfloat16(v); }++template <> __device__ __forceinline__ half mmq_cast<half>(float v) { return __float2half(v); }++++struct mmq_dst {++ void * p;++ ggml_type type;++};+++ typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);+ typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);+ typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,+@@ -429,8 +440,8 @@+ #include "mmq-load-tiles.cuh"+ #include "mmq-vec-dot.cuh"++-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(+- const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, float * __restrict__ dst,++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(++ const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, dst_t * __restrict__ dst,+ const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;+@@ -456,21 +467,21 @@++ if constexpr (type == GGML_TYPE_NVFP4) {+ if (y_scale_used) {+- dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);+ } else {+- dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);+ }+ } else {+- dst[ids_dst[j]*stride + i] = sum[(j0/nwarps) * (I/warp_size) + i0/warp_size];++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);+ GGML_UNUSED(y_scale_used);+ }+ }+ }+ }++-template<ggml_type type, int J, bool fallback, ggml_prec prec_src1>++template<ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float>+ static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(+- const float * __restrict__ sum, const int * __restrict__ ids_dst, float * __restrict__ dst,++ const float * __restrict__ sum, const int * __restrict__ ids_dst, dst_t * __restrict__ dst,+ const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {++ #if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+@@ -506,12 +517,12 @@++ if constexpr (type == GGML_TYPE_NVFP4) {+ if (y_scale_used) {+- dst[ids_dst[j]*stride + i] = y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l];++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]);+ } else {+- dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);+ }+ } else {+- dst[ids_dst[j]*stride + i] = sum[(j0/tile_C::J + n)*tile_C::ne + l];++ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);+ GGML_UNUSED(y_scale_used);+ }+ }+@@ -869,10 +880,10 @@++ // ---------------------------------------------------------------------------------------------++-template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1>++template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1, typename dst_t>+ static __device__ __forceinline__ void mul_mat_q_process_tile(+ const char * __restrict__ x, const int offset_x, const int * __restrict__ y,+- const int * __restrict__ ids_dst, float * __restrict__ dst, float * __restrict__ tmp_fixup,++ const int * __restrict__ ids_dst, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,+ const float * __restrict__ y_scale,+ const int stride_row_x, const int ncols_y, const int stride_col_dst,+ const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {+@@ -942,19 +953,32 @@++ if (fixup) {+ write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);+- } else {++ } else if constexpr (std::is_same_v<dst_t, float>) {+ write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);++ } else if constexpr (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) {++ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);++ } else {++ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);+ }+ }++++// A flat MoE grid runs only the (expert, column tile) pairs listed in tile_map, see ggml_cuda_launch_mm_ids_tile_map.++static __device__ __forceinline__ void mmq_flat_tile(const int2 * __restrict__ tile_map, int & zt, int & jt) {++ if (tile_map) {++ const int2 t = tile_map[jt];++ zt = t.x;++ jt = t.y;++ }++}+++ // The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598++-template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t>+ __launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1))+ static __global__ void mul_mat_q(+ const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,+- const int32_t * __restrict__ expert_bounds, float * __restrict__ dst, float * __restrict__ tmp_fixup,++ const int32_t * __restrict__ expert_bounds, const int2 * __restrict__ tile_map, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,+ const float * __restrict__ y_scale,+ const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,+ const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,+@@ -993,9 +1017,10 @@+ if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {+ const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);+ const int wt = tmp2.x;+- const int zt = tmp2.y;+- const int jt = blockIdx.y;++ int zt = tmp2.y;++ int jt = blockIdx.y;+ const int it = blockIdx.x;++ mmq_flat_tile(tile_map, zt, jt);++ // Defaults for regular matrix multiplication:+ int col_low = 0;+@@ -1076,14 +1101,15 @@+ while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) {+ int tmp = fastdiv(kbc, blocks_per_ne00);+ uint2 tmp2 = fast_div_modulo(tmp, ntx);+- const int jt = tmp2.y;++ int jt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nchannels_y);+- const int zt = tmp2.y;++ int zt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nsamples_y);+ const int wt = tmp2.y;+ const int it = tmp2.x;++ mmq_flat_tile(tile_map, zt, jt);++ // Defaults for regular matrix multiplication:+ int col_low = 0;+@@ -1165,14 +1191,15 @@++ int tmp = fastdiv(kbc, blocks_per_ne00);+ uint2 tmp2 = fast_div_modulo(tmp, ntx);+- const int jt = tmp2.y;++ int jt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nchannels_y);+- const int zt = tmp2.y;++ int zt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nsamples_y);+ const int wt = tmp2.y;+ const int it = tmp2.x;++ mmq_flat_tile(tile_map, zt, jt);++ // Defaults for regular matrix multiplication:+ int col_low = 0;+@@ -1375,14 +1402,26 @@+ }+ }+++// FreeToken: copies an fp32 result with dst's layout into dst++template <typename dst_t>++static __global__ void mmq_store_dst(const float * __restrict__ src, dst_t * __restrict__ dst, const int nrows, const int stride) {++ const int64_t col = blockIdx.x;++ const int row = blockIdx.y*blockDim.x + threadIdx.x;++ if (row < nrows) {++ dst[col*stride + row] = mmq_cast<dst_t>(src[col*stride + row]);++ }++}+++ struct mmq_args {+- const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; float * dst;++ const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; mmq_dst dst;+ const float * y_scale;+ int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst;+ int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst;+ int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;+ int64_t ncols_max;+ int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance.++ const int2 * tile_map = nullptr; // Flat MoE grid: the (expert, column tile) of each of the ntx_flat grid columns.++ int ntx_flat = 0;+ };++ static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {+@@ -1406,12 +1445,11 @@++ const dim3 block_dims(warp_size, nwarps, 1);++- CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1>), nbytes_shared);+- CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1>), nbytes_shared);+-++ // a flat MoE grid puts all experts into one channel of ntx_flat column tiles++ const int64_t nchannels_grid = args.tile_map ? 1 : args.nchannels_y;+ const int nty = (args.nrows_x + config.I - 1) / config.I;+- const int ntx = (args.ncols_max + config.J - 1) / config.J;+- const int ntzw = args.nchannels_y * args.nsamples_y;++ const int ntx = args.tile_map ? args.ntx_flat : (args.ncols_max + config.J - 1) / config.J;++ const int ntzw = nchannels_grid * args.nsamples_y;+ const dim3 block_nums_xy_tiling(nty, ntx, ntzw);++ GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);+@@ -1421,18 +1459,71 @@++ const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);+ const uint3 ntx_fd = init_fastdiv_values(ntx);+- const uint3 nchannels_y_fd = init_fastdiv_values(args.nchannels_y);++ const uint3 nchannels_y_fd = init_fastdiv_values(nchannels_grid);+ const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y);+ const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio);+ const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);++- if (!config.stream_k) {+- mul_mat_q<type, J, fallback, prec_src1><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>+- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, nullptr, args.y_scale,++ const auto run = [&](auto * dst, const dim3 & block_nums, float * tmp_fixup) {++ using dst_t = std::remove_pointer_t<decltype(dst)>;++ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1, dst_t>), nbytes_shared);++ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1, dst_t>), nbytes_shared);++ mul_mat_q<type, J, fallback, prec_src1, dst_t><<<block_nums, block_dims, nbytes_shared, stream>>>++ (args.x, args.y, args.ids_dst, args.expert_bounds, args.tile_map, dst, tmp_fixup, args.y_scale,+ blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,+ channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,+ sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,+ ntx_fd);++ };++++ // FreeToken: the fixup kernel adds partial sums into an fp32 dst, so a bf16 dst that needs it is computed in fp32 first.++ // An fp16 dst always is, unless GGML_MMQ_F16_DST builds the fp16 kernels.++ const auto launch = [&](const dim3 & block_nums, const bool fixup, float * tmp_fixup) {++ bool via_f32 = fixup && args.dst.type != GGML_TYPE_F32;++#ifndef GGML_MMQ_F16_DST++ via_f32 = via_f32 || args.dst.type == GGML_TYPE_F16;++#endif // GGML_MMQ_F16_DST++ ggml_cuda_pool_alloc<float> dst_f32(ctx.pool(id));++ float * dst_float = args.dst.type == GGML_TYPE_F32 ? (float *) args.dst.p : nullptr;++ if (via_f32) {++ GGML_ASSERT(args.ids_dst || (args.nchannels_y == 1 && args.nsamples_y == 1));++ dst_float = dst_f32.alloc(args.ncols_dst * args.nrows_dst);++ }++++ if (dst_float) {++ run(dst_float, block_nums, tmp_fixup);++ } else if (args.dst.type == GGML_TYPE_BF16) {++ run((nv_bfloat16 *) args.dst.p, block_nums, tmp_fixup);++ }++#ifdef GGML_MMQ_F16_DST++ else {++ run((half *) args.dst.p, block_nums, tmp_fixup);++ }++#endif // GGML_MMQ_F16_DST++++ if (fixup) {++ CUDA_CHECK(cudaGetLastError());++ const dim3 block_nums_fixup(block_nums.x, config.I/warp_size, 1);++ const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);++ mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>>++ (args.ids_dst, args.expert_bounds, dst_float, tmp_fixup, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,++ args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,++ ntx_fd);++ }++++ if (via_f32) {++ CUDA_CHECK(cudaGetLastError());++ const dim3 block_nums_store(args.ncols_dst, (args.nrows_x + 255) / 256, 1);++ if (args.dst.type == GGML_TYPE_BF16) {++ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (nv_bfloat16 *) args.dst.p, args.nrows_x, args.nrows_dst);++ } else {++ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (half *) args.dst.p, args.nrows_x, args.nrows_dst);++ }++ }++ };++++ if (!config.stream_k) {++ launch(block_nums_xy_tiling, false, nullptr);+ return;+ }++@@ -1441,7 +1532,8 @@+ const int ntiles_dst = ntx * nty * ntzw;+ const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;+ const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);+- const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1);++ // The fixup kernel does not read tile_map, so a flat MoE grid always runs one tile per block.++ const dim3 block_nums_stream_k(args.tile_map || (GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90) ? ntiles_dst : nsm, 1, 1);++ GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.++@@ -1453,25 +1545,7 @@+ tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);+ }++- const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1);+- const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);+-+- mul_mat_q<type, J, fallback, prec_src1><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>+- (args.x, args.y, args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, args.y_scale,+- blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,+- channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,+- sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,+- ntx_fd);+-+- if (!fixup_needed) {+- return;+- }+-+- CUDA_CHECK(cudaGetLastError());+- mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>>+- (args.ids_dst, args.expert_bounds, args.dst, tmp_fixup.ptr, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,+- args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,+- ntx_fd);++ launch(block_nums_stream_k, fixup_needed, tmp_fixup.ptr);+ }++ template <ggml_type type, bool fallback, ggml_prec prec_src1>+@@ -1579,7 +1653,9 @@++ // -------------------------------------------------------------------------------------------------------------------------+++// FreeToken: under ids, ncols_max > 0 bounds the tokens any one expert gets and sizes the tile grid; otherwise the grid is flat, see ggml_cuda_launch_mm_ids_tile_map+ void ggml_cuda_mul_mat_q(+- ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);++ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,++ int64_t ncols_max = -1);++ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);+--- a/ggml/src/ggml-cuda/mmvq.cu++++ b/ggml/src/ggml-cuda/mmvq.cu+@@ -1,4 +1,5 @@+ #include "mmvq.cuh"++#include "convert.cuh"+ #include "quantize.cuh"+ #include "unary.cuh"+ #include "vecdotq.cuh"+@@ -596,10 +597,11 @@+ return 1;+ }++-template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false>++// FreeToken: dst_t is float, nv_bfloat16 or half++template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false, typename dst_t = float>+ __launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1)+ static __global__ void mul_mat_vec_q(+- const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,++ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst_ptr,+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,+ uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,+ const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,+@@ -608,7 +610,7 @@+ const void * GGML_CUDA_RESTRICT vx = vx_ptr;+ const void * GGML_CUDA_RESTRICT vy = vy_ptr;+ const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;+- float * GGML_CUDA_RESTRICT dst = dst_ptr;++ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;++ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int qi = ggml_cuda_type_traits<type>::qi;+@@ -629,7 +631,9 @@+ const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y;+ if (shared_expert) {+ vx = fusion.shared_up;+- dst = fusion.shared_dst;++ if constexpr (std::is_same_v<dst_t, float>) {++ dst = fusion.shared_dst;++ }+ stride_col_dst = fusion.shared_stride_col_dst;+ }++@@ -837,7 +841,7 @@+ }+ }+ }+- dst[j*stride_col_dst + i] = result;++ dst[j*stride_col_dst + i] = ggml_cuda_cast<dst_t>(result);+ }+ }+ }+@@ -854,11 +858,11 @@+ // Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst)+ // Block: (warp_size, ncols_dst) - each warp handles one token independently.+ // No shared memory reduction needed since each warp works alone.+-template <ggml_type type, int c_rows_per_block, bool has_fusion = false>++template <ggml_type type, int c_rows_per_block, bool has_fusion = false, typename dst_t = float>+ __launch_bounds__(get_mmvq_mmid_max_batch_for_device<type>()*ggml_cuda_get_physical_warp_size(), 1)+ static __global__ void mul_mat_vec_q_moe(+ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion,+- float * dst_ptr,++ dst_t * dst_ptr,+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,+ const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst,+ const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,+@@ -866,7 +870,7 @@+ const void * GGML_CUDA_RESTRICT vx = vx_ptr;+ const void * GGML_CUDA_RESTRICT vy = vy_ptr;+ const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;+- float * GGML_CUDA_RESTRICT dst = dst_ptr;++ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;++ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int qi = ggml_cuda_type_traits<type>::qi;+@@ -878,7 +882,9 @@+ const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1;+ if (shared_expert) {+ vx = fusion.shared_up;+- dst = fusion.shared_dst;++ if constexpr (std::is_same_v<dst_t, float>) {++ dst = fusion.shared_dst;++ }+ stride_col_dst = fusion.shared_stride_col_dst;+ }++@@ -998,7 +1004,7 @@+ }+ }+ }+- dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = result;++ dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = ggml_cuda_cast<dst_t>(result);+ }++ if constexpr (!has_fusion) {+@@ -1020,9 +1026,9 @@+ return {block_nums, block_dims};+ }++-template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false>++template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false, typename dst_t>+ static void mul_mat_vec_q_switch_fusion(+- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,++ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,+ const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,+ const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,+@@ -1035,7 +1041,7 @@+ if constexpr (c_ncols_dst == 1) {+ if (has_fusion) {+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);+- ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters>, launch_params,++ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters, dst_t>, launch_params,+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);+@@ -1046,15 +1052,15 @@+ GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1");++ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);+- ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters>, launch_params,++ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters, dst_t>, launch_params,+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);+ }++-template <ggml_type type>++template <ggml_type type, typename dst_t>+ static void mul_mat_vec_q_moe_launch(+- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,++ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,+ const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,+ const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,+@@ -1071,13 +1077,13 @@+ fusion.x_scale != nullptr || fusion.gate_scale != nullptr;++ if (has_fusion) {+- ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true>, launch_params,++ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true, dst_t>, launch_params,+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,+ stride_row_x, stride_col_y, stride_col_dst,+ stride_channel_x, stride_channel_y, stride_channel_dst,+ ncols_dst, ids_stride);+ } else {+- ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false>, launch_params,++ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false, dst_t>, launch_params,+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,+ stride_row_x, stride_col_y, stride_col_dst,+ stride_channel_x, stride_channel_y, stride_channel_dst,+@@ -1085,9 +1091,9 @@+ }+ }++-template <ggml_type type>++template <ggml_type type, typename dst_t>+ static void mul_mat_vec_q_switch_ncols_dst(+- const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,++ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,+ const int ncols_x, const int nrows_x, const int ncols_dst,+ const int stride_row_x, const int stride_col_y, const int stride_col_dst,+ const int nchannels_x, const int nchannels_y, const int nchannels_dst,+@@ -1277,8 +1283,10 @@+ GGML_ABORT("fatal error");+ }+ }++++template <typename dst_t>+ static void mul_mat_vec_q_switch_type(+- const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, float * dst,++ const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,+ const int ncols_x, const int nrows_x, const int ncols_dst,+ const int stride_row_x, const int stride_col_y, const int stride_col_dst,+ const int nchannels_x, const int nchannels_y, const int nchannels_dst,+@@ -1431,9 +1439,10 @@++ void ggml_cuda_mul_mat_vec_q(+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,+- const ggml_cuda_mm_fusion_args_host * fusion) {+- GGML_ASSERT( src1->type == GGML_TYPE_F32);+- GGML_ASSERT( dst->type == GGML_TYPE_F32);++ const ggml_cuda_mm_fusion_args_host * fusion, const void * src1_q8_1_pre) {++ // FreeToken: activations and results may also be bf16 or fp16++ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);++ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.++ GGML_TENSOR_BINARY_OP_LOCALS;+@@ -1526,12 +1535,22 @@+ }++ const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);+- ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);+- {++ // FreeToken: src1_q8_1_pre is src1 already quantized by quantize_row_q8_1_cuda with this padding and layout++ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool());++ const void * src1_q8 = src1_q8_1_pre;++ if (src1_q8 == nullptr) {++ src1_q8_1.alloc(ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);++ src1_q8 = src1_q8_1.get();+ const int64_t s11 = src1->nb[1] / ts_src1;+ const int64_t s12 = src1->nb[2] / ts_src1;+ const int64_t s13 = src1->nb[3] / ts_src1;+- quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);++ if (src1->type == GGML_TYPE_BF16) {++ quantize_row_q8_1_cuda((const nv_bfloat16 *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);++ } else if (src1->type == GGML_TYPE_F16) {++ quantize_row_q8_1_cuda((const half *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);++ } else {++ quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);++ }+ }++ const int64_t s01 = src0->nb[1] / ts_src0;+@@ -1556,11 +1575,23 @@++ const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;++- mul_mat_vec_q_switch_type(+- src0->data, src0->type, src1_q8_1.get(), ids_d, fusion_local, dst_d, ne00,+- ne01, ncols_dst, s01, stride_col_y, stride_col_dst,+- ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,+- ne03, ne3, s03, s13, s3, ids_stride, stream);++ const auto launch = [&](auto * dst_ptr) {++ mul_mat_vec_q_switch_type(++ src0->data, src0->type, src1_q8, ids_d, fusion_local, dst_ptr, ne00,++ ne01, ncols_dst, s01, stride_col_y, stride_col_dst,++ ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,++ ne03, ne3, s03, s13, s3, ids_stride, stream);++ };++ if (dst->type == GGML_TYPE_BF16) {++ // FreeToken: the fused shared expert writes an fp32 shared_dst++ GGML_ASSERT(!fusion || !fusion->shared_up);++ launch((nv_bfloat16 *) dst->data);++ } else if (dst->type == GGML_TYPE_F16) {++ GGML_ASSERT(!fusion || !fusion->shared_up);++ launch((half *) dst->data);++ } else {++ launch(dst_d);++ }+ }++ void ggml_cuda_op_mul_mat_vec_q(+--- a/ggml/src/ggml-cuda/mmvq.cuh++++ b/ggml/src/ggml-cuda/mmvq.cuh+@@ -8,8 +8,10 @@+ // based on the quantization type and GPU architecture (compute capability).+ int get_mmvq_mmid_max_batch(ggml_type type, int cc);+++// FreeToken: src1_q8_1_pre skips the internal quantization of src1+ void ggml_cuda_mul_mat_vec_q(ggml_backend_cuda_context & ctx,+- const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr);++ const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr,++ const void * src1_q8_1_pre = nullptr);++ void ggml_cuda_op_mul_mat_vec_q(+ ggml_backend_cuda_context & ctx,+--- a/ggml/src/ggml-cuda/quantize.cu++++ b/ggml/src/ggml-cuda/quantize.cu+@@ -1,4 +1,5 @@+ #include "quantize.cuh"++#include "convert.cuh"+ #include <cstdint>++ #if defined(BLACKWELL_MMA_AVAILABLE)+@@ -50,13 +51,14 @@+ #endif // CUDART_VERSION >= 12080+ #endif // defined(BLACKWELL_MMA_AVAILABLE)+++template <typename src_t>+ __launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1)+ static __global__ void quantize_q8_1(+- const float * x_ptr, void * vy_ptr,++ const src_t * x_ptr, void * vy_ptr,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const uint32_t ne1, const uint3 ne2) {+ ggml_cuda_pdl_lc();+- const float * GGML_CUDA_RESTRICT x = x_ptr;++ const src_t * GGML_CUDA_RESTRICT x = x_ptr;+ void * GGML_CUDA_RESTRICT vy = vy_ptr;+ const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;++@@ -81,7 +83,7 @@+ const int64_t iqs = i_cont % QK8_1; // quant index++ ggml_cuda_pdl_sync();+- const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f;++ const float xi = i0 < ne00 ? ggml_cuda_cast<float>(x[i03*s03 + i02*s02 + i01*s01 + i00]) : 0.0f;+ float amax = fabsf(xi);+ float sum = xi;++@@ -452,10 +454,29 @@+ GGML_UNUSED(n_expert_used);+ }+++// FreeToken: 4 consecutive activations as float, from fp32, bf16 or fp16++static __device__ __forceinline__ float4 load_float4(const float * x) {++ return *(const float4 *) x;++}++++static __device__ __forceinline__ float4 load_float4(const nv_bfloat16 * x) {++ const uint2 v = *(const uint2 *) x;++ const float2 lo = __bfloat1622float2(*(const nv_bfloat162 *) &v.x);++ const float2 hi = __bfloat1622float2(*(const nv_bfloat162 *) &v.y);++ return make_float4(lo.x, lo.y, hi.x, hi.y);++}++++static __device__ __forceinline__ float4 load_float4(const half * x) {++ const uint2 v = *(const uint2 *) x;++ const float2 lo = __half22float2(*(const half2 *) &v.x);++ const float2 hi = __half22float2(*(const half2 *) &v.y);++ return make_float4(lo.x, lo.y, hi.x, hi.y);++}+++ // scatter: grid over tokens, quantize once, write to all the token's compact rows+-template <mmq_q8_1_ds_layout ds_layout, bool scatter>++template <mmq_q8_1_ds_layout ds_layout, bool scatter, typename src_t>+ static __global__ void quantize_mmq_q8_1(+- const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,++ const src_t * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) {++@@ -481,14 +502,13 @@+ base_idx = i3*s03 + i2*s02 + i01*s01;+ }++- const float4 * x4 = (const float4 *) x;+ block_q8_1_mmq * y = (block_q8_1_mmq *) vy;++ const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel+ const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block++ // Load 4 floats per thread and calculate max. abs. value between them:+- const float4 xi = i0 < ne00 ? x4[(base_idx + i00)/4] : make_float4(0.0f, 0.0f, 0.0f, 0.0f);++ const float4 xi = i0 < ne00 ? load_float4(x + base_idx + i00) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);+ float amax = fabsf(xi.x);+ amax = fmaxf(amax, fabsf(xi.y));+ amax = fmaxf(amax, fabsf(xi.z));+@@ -554,8 +574,9 @@+ GGML_UNUSED(n_expert_used);+ }+++template <typename src_t>+ void quantize_row_q8_1_cuda(+- const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,++ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {+ GGML_ASSERT(!ids);+@@ -567,12 +588,13 @@+ const dim3 num_blocks(block_num_x, ne1, ne2*ne3);+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream);+- ggml_cuda_kernel_launch(quantize_q8_1, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);++ ggml_cuda_kernel_launch(quantize_q8_1<src_t>, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);+ GGML_UNUSED(type_src0);+ }+++template <typename src_t>+ void quantize_mmq_q8_1_cuda(+- const float * x, const int32_t * ids, void * vy, const ggml_type type_src0,++ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {+ GGML_ASSERT(ne00 % 4 == 0);+@@ -584,15 +606,15 @@+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);+ switch (mmq_get_q8_1_ds_layout(type_src0)) {+ case MMQ_Q8_1_DS_LAYOUT_D4:+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, false>++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, false, src_t>+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ break;+ case MMQ_Q8_1_DS_LAYOUT_DS4:+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, false>++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, false, src_t>+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ break;+ case MMQ_Q8_1_DS_LAYOUT_D2S6:+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, false>++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, false, src_t>+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ break;+ default:+@@ -601,8 +623,9 @@+ }++ // scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)++template <typename src_t>+ void quantize_scatter_mmq_q8_1_cuda(+- const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,++ const src_t * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,+ const int64_t ne00, const int64_t stride_token, const int64_t ne0,+ const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) {+ GGML_ASSERT(ne00 % 4 == 0);+@@ -613,15 +636,15 @@+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);+ switch (mmq_get_q8_1_ds_layout(type_src0)) {+ case MMQ_Q8_1_DS_LAYOUT_D4:+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, true><<<num_blocks, block_size, 0, stream>>>(++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, true, src_t><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);+ break;+ case MMQ_Q8_1_DS_LAYOUT_DS4:+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, true><<<num_blocks, block_size, 0, stream>>>(++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, true, src_t><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);+ break;+ case MMQ_Q8_1_DS_LAYOUT_D2S6:+- quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, true><<<num_blocks, block_size, 0, stream>>>(++ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, true, src_t><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);+ break;+ default:+@@ -692,3 +715,15 @@+ quantize_mmq_mxfp4<false><<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ }+ }++++#define FREETOKEN_QUANTIZE_INSTANCES(src_t) \++ template void quantize_row_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \++ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \++ template void quantize_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \++ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \++ template void quantize_scatter_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, \++ int64_t, int64_t, int64_t, int64_t, int, cudaStream_t);++++FREETOKEN_QUANTIZE_INSTANCES(float)++FREETOKEN_QUANTIZE_INSTANCES(nv_bfloat16)++FREETOKEN_QUANTIZE_INSTANCES(half)+--- a/ggml/src/ggml-cuda/quantize.cuh++++ b/ggml/src/ggml-cuda/quantize.cuh+@@ -16,13 +16,16 @@+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);+++// FreeToken: src_t is float, nv_bfloat16 or half++template <typename src_t>+ void quantize_row_q8_1_cuda(+- const float * x, const int32_t * ids, void * vy,++ const src_t * x, const int32_t * ids, void * vy,+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);+++template <typename src_t>+ void quantize_mmq_q8_1_cuda(+- const float * x, const int32_t * ids, void * vy,++ const src_t * x, const int32_t * ids, void * vy,+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);++@@ -57,7 +60,8 @@+ int n_expert_used,+ cudaStream_t stream);++-void quantize_scatter_mmq_q8_1_cuda(const float * x,++template <typename src_t>++void quantize_scatter_mmq_q8_1_cuda(const src_t * x,+ const int32_t * ids_src1_inv,+ void * vy,+ ggml_type type_src0,diff --git a/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cu b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cunew file mode 100644index 00000000..cc735c3b--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_ops.cu@@ -0,0 +1,426 @@+// tvm-ffi ops over the vendored ggml-cuda kernels. Weights stay in their packed ggml blocks.+// Activations go through the same q8_1 quantization and kernel choice as llama.cpp's mul_mat / mul_mat_id.+// Activations are fp32, bf16 or fp16 with contiguous rows. Results have the activation dtype and are accumulated in fp32; the caller allocates every output.++#include <freetoken/utils.h>++#include "common.cuh"+#include "convert.cuh"+#include "getrows.cuh"+#include "mmq.cuh"+#include "mmvq.cuh"+#include "quantize.cuh"++#include <tvm/ffi/container/tensor.h>+#include <tvm/ffi/extra/c_env_api.h>+#include <tvm/ffi/function.h>+#include <tvm/ffi/optional.h>+#include <tvm/ffi/string.h>++#include <algorithm>+#include <array>+#include <memory>+#include <string>++using tvm::ffi::Optional;+using tvm::ffi::String;+using tvm::ffi::TensorView;++namespace {++using host::RuntimeCheck;++struct DeviceGuard {+ int prev, device;++ explicit DeviceGuard(DLDevice d) : prev(ggml_cuda_get_device()), device(d.device_id) {+#ifdef GGML_USE_HIP+ RuntimeCheck(d.device_type == kDLROCM, "ggml ops take ROCm tensors");+#else+ RuntimeCheck(d.device_type == kDLCUDA, "ggml ops take CUDA tensors");+#endif+ ggml_cuda_set_device(device);+ }++ ~DeviceGuard() {+ if (prev != device) {+ cudaSetDevice(prev);+ }+ }+};++// ggml treats a null stream as unset and would create its own; name the default stream explicitly+cudaStream_t current_stream(DLDevice d) {+ auto stream = static_cast<cudaStream_t>(TVMFFIEnvGetStream(d.device_type, d.device_id));+#ifdef GGML_USE_HIP+ return stream == nullptr ? hipStreamLegacy : stream;+#else+ return stream == nullptr ? cudaStreamLegacy : stream;+#endif+}++ggml_backend_cuda_context & context(DLDevice d) {+ static std::unique_ptr<ggml_backend_cuda_context> ctxs[GGML_CUDA_MAX_DEVICES];+ const int dev = d.device_id;+ if (!ctxs[dev]) {+ ctxs[dev] = std::make_unique<ggml_backend_cuda_context>(dev);+ }+ ctxs[dev]->curr_stream_no = 0;+ ctxs[dev]->streams[dev][0] = current_stream(d);+ return *ctxs[dev];+}++ggml_tensor describe(ggml_type type, const void * data, std::array<int64_t, 4> ne, std::array<size_t, 4> nb) {+ ggml_tensor t{};+ t.type = type;+ t.data = const_cast<void *>(data);+ for (int i = 0; i < 4; ++i) {+ t.ne[i] = ne[i];+ t.nb[i] = nb[i];+ }+ return t;+}++int device_cc() { return ggml_cuda_info().devices[ggml_cuda_get_device()].cc; }++bool is_dtype(DLDataType dt, DLDataTypeCode code, int bits) { return dt.code == code && dt.bits == bits && dt.lanes == 1; }++ggml_type float_type(TensorView t) {+ const DLDataType dt = t.dtype();+ if (is_dtype(dt, kDLFloat, 32)) {+ return GGML_TYPE_F32;+ }+ if (is_dtype(dt, kDLBfloat, 16)) {+ return GGML_TYPE_BF16;+ }+ RuntimeCheck(is_dtype(dt, kDLFloat, 16), "ggml tensors must be fp32, bf16 or fp16");+ return GGML_TYPE_F16;+}++// the activation dtype, which the result must share+ggml_type io_type(TensorView x, TensorView out) {+ const ggml_type type = float_type(x);+ RuntimeCheck(float_type(out) == type, "the result must have the activation dtype");+ return type;+}++int64_t check_weight(TensorView w, ggml_type type, int64_t k) {+ RuntimeCheck(is_dtype(w.dtype(), kDLUInt, 8), "ggml weight must be a uint8 tensor");+ RuntimeCheck(w.stride(w.ndim() - 1) == 1, "ggml weight rows must be contiguous");+ const int64_t bs = ggml_blck_size(type);+ RuntimeCheck(k % bs == 0, "K=", k, " is not a multiple of the ", ggml_type_name(type), " block size ", bs);+ const int64_t row_bytes = k / bs * (int64_t) ggml_type_size(type);+ RuntimeCheck(w.size(w.ndim() - 1) == row_bytes, "ggml weight row is ", w.size(w.ndim() - 1), " bytes, ", ggml_type_name(type),+ " K=", k, " needs ", row_bytes);+ return row_bytes;+}++int64_t q8_1_row_bytes(int64_t k) { return GGML_PAD(k, MATRIX_ROW_PADDING) / QK8_1 * (int64_t) sizeof(block_q8_1); }++// MMQ tiles read up to MATRIX_ROW_PADDING values past the last row; the caller keeps this many bytes after it+int64_t mmq_tail_bytes(int64_t type_id, int64_t k) {+ const int64_t rem = k % MATRIX_ROW_PADDING;+ return rem == 0 ? 0 : (int64_t) ggml_row_size((ggml_type) type_id, MATRIX_ROW_PADDING - rem);+}++struct Activation {+ const void * data;+ ggml_type type;+ size_t esz;+};++// The native fp4 MMQ path quantizes fp32 activations only; it is built only with GGML_CUDA_FP4_MMA.+bool mmq_reads_16bit(ggml_type type) {+#ifdef GGML_CUDA_FP4_MMA+ return type != GGML_TYPE_MXFP4 && type != GGML_TYPE_NVFP4;+#else+ GGML_UNUSED(type);+ return true;+#endif+}++// x as the kernel reads it: x itself, or an fp32 copy in ``tmp`` for a kernel that reads fp32 only+Activation activation(TensorView x, bool f16_ok, ggml_cuda_pool_alloc<float> & tmp, cudaStream_t stream) {+ RuntimeCheck(x.IsContiguous(), "ggml activations must be contiguous");+ const ggml_type type = float_type(x);+ if (type == GGML_TYPE_F32) {+ return {x.data_ptr(), type, 4};+ }+ if (f16_ok) {+ return {x.data_ptr(), type, 2};+ }+ float * f32 = tmp.alloc(x.numel());+ ggml_get_to_fp32_cuda(type)(x.data_ptr(), f32, x.numel(), stream);+ return {f32, GGML_TYPE_F32, 4};+}++} // namespace++// out[M, q8_1_row_bytes(K)] = x[M, K] as MMVQ's q8_1 blocks, for GEMVs that share one input+void op_quantize_q8_1(TensorView x, TensorView out) {+ RuntimeCheck(x.ndim() == 2 && x.stride(1) == 1, "quantize_q8_1 takes x[M, K] with contiguous rows");+ const DeviceGuard guard(x.device());+ const int64_t m = x.size(0), k = x.size(1), k_padded = GGML_PAD(k, MATRIX_ROW_PADDING);+ RuntimeCheck(is_dtype(out.dtype(), kDLUInt, 8) && out.IsContiguous() && out.ndim() == 2 && out.size(0) == m &&+ out.size(1) == q8_1_row_bytes(k),+ "out must be a contiguous uint8 [M, q8_1_row_bytes(K)] tensor");+ if (m == 0) {+ return;+ }+ const int64_t s1 = x.stride(0);+ const cudaStream_t stream = current_stream(x.device());+ const ggml_type type = float_type(x);+ if (type == GGML_TYPE_BF16) {+ quantize_row_q8_1_cuda((const nv_bfloat16 *) x.data_ptr(), nullptr, out.data_ptr(), GGML_TYPE_Q8_0, k, s1, s1 * m, s1 * m, k_padded, m, 1, 1, stream);+ } else if (type == GGML_TYPE_F16) {+ quantize_row_q8_1_cuda((const half *) x.data_ptr(), nullptr, out.data_ptr(), GGML_TYPE_Q8_0, k, s1, s1 * m, s1 * m, k_padded, m, 1, 1, stream);+ } else {+ quantize_row_q8_1_cuda((const float *) x.data_ptr(), nullptr, out.data_ptr(), GGML_TYPE_Q8_0, k, s1, s1 * m, s1 * m, k_padded, m, 1, 1, stream);+ }+}++// out[M, N] = x[M, K] @ W[N, K]^T; out may be a view with strided rows; MMVQ reads ``xq`` (op_quantize_q8_1 of x) instead of quantizing x again+void op_mul_mat(TensorView w, int64_t type_id, TensorView x, TensorView out, Optional<TensorView> xq) {+ RuntimeCheck(x.ndim() == 2 && w.ndim() == 2, "mul_mat takes w[N, row_bytes] and x[M, K]");+ const DeviceGuard guard(x.device());+ const ggml_type type = (ggml_type) type_id;+ const int64_t m = x.size(0), k = x.size(1), n = w.size(0);+ check_weight(w, type, k);+ RuntimeCheck(out.ndim() == 2 && out.size(0) == m && out.size(1) == n && out.stride(1) == 1, "mul_mat out must be a [M, N] view with contiguous rows");+ const ggml_type otype = io_type(x, out);+ if (m == 0) {+ return;+ }+ const size_t s1 = w.stride(0);+ ggml_tensor src0 = describe(type, w.data_ptr(), {k, n, 1, 1}, {ggml_type_size(type), s1, s1 * n, s1 * n});+ const size_t oe = ggml_type_size(otype), os = out.stride(0) * oe;+ ggml_tensor dst = describe(otype, out.data_ptr(), {n, m, 1, 1}, {oe, os, os * m, os * m});+ auto & ctx = context(x.device());+ const cudaStream_t stream = ctx.stream();+ const int cc = device_cc();+ ggml_cuda_pool_alloc<float> x32(ctx.pool());+ if (ggml_cuda_should_use_mmvq(type, cc, m)) {+ const Activation a = activation(x, true, x32, stream);+ ggml_tensor src1 = describe(a.type, a.data, {k, m, 1, 1}, {a.esz, a.esz * k, a.esz * k * m, a.esz * k * m});+ const void * q8 = nullptr;+ if (xq.has_value()) {+ const TensorView q = xq.value();+ RuntimeCheck(is_dtype(q.dtype(), kDLUInt, 8) && q.IsContiguous() && q.ndim() == 2 && q.size(0) == m && q.size(1) == q8_1_row_bytes(k),+ "xq must be quantize_q8_1 of x");+ q8 = q.data_ptr();+ }+ ggml_cuda_mul_mat_vec_q(ctx, &src0, &src1, nullptr, &dst, nullptr, q8);+ } else if (ggml_cuda_should_use_mmq(type, cc, m, 0)) {+ const Activation a = activation(x, mmq_reads_16bit(type), x32, stream);+ ggml_tensor src1 = describe(a.type, a.data, {k, m, 1, 1}, {a.esz, a.esz * k, a.esz * k * m, a.esz * k * m});+ ggml_cuda_mul_mat_q(ctx, &src0, &src1, nullptr, &dst);+ } else {+ RuntimeCheck(false, "no ggml kernel for ", ggml_type_name(type), " with M=", m);+ }+}++// out[T, k, N]: row (t, j) multiplies x[t, j or 0] by expert ids[t, j] of W[E, N, K]; the matrix kernel runs a flat grid over the tiles that hold routes+void op_mul_mat_id(TensorView w, int64_t type_id, TensorView x, TensorView ids, TensorView out) {+ RuntimeCheck(w.ndim() == 3, "mul_mat_id takes w[E, N, row_bytes]");+ RuntimeCheck(is_dtype(ids.dtype(), kDLInt, 32) && ids.ndim() == 2 && ids.IsContiguous(), "ids must be a contiguous int32 [T, k] tensor");+ const DeviceGuard guard(x.device());+ const ggml_type type = (ggml_type) type_id;+ const int64_t t = ids.size(0), top_k = ids.size(1);+ RuntimeCheck(x.ndim() == 3 && x.size(0) == t && (x.size(1) == 1 || x.size(1) == top_k), "x must be [T, 1, K] or [T, k, K]");+ const int64_t ky = x.size(1), k = x.size(2), e = w.size(0), n = w.size(1);+ check_weight(w, type, k);+ RuntimeCheck(out.ndim() == 3 && out.size(0) == t && out.size(1) == top_k && out.size(2) == n && out.IsContiguous(),+ "out must be a contiguous [T, k, N] tensor");+ const ggml_type otype = io_type(x, out);+ if (t == 0) {+ return;+ }+ const size_t s1 = w.stride(1), s2 = w.stride(0);+ ggml_tensor src0 = describe(type, w.data_ptr(), {k, n, e, 1}, {ggml_type_size(type), s1, s2, s2 * e});+ const size_t ids_stride = sizeof(int32_t) * top_k, ids_bytes = ids_stride * t;+ ggml_tensor tids = describe(GGML_TYPE_I32, ids.data_ptr(), {top_k, t, 1, 1}, {sizeof(int32_t), ids_stride, ids_bytes, ids_bytes});+ auto & ctx = context(x.device());+ const cudaStream_t stream = ctx.stream();+ const int cc = device_cc();+ const bool mmvq = t <= MMVQ_MAX_BATCH_SIZE && t <= get_mmvq_mmid_max_batch(type, cc);+ RuntimeCheck(mmvq || ggml_cuda_should_use_mmq(type, cc, t, e), "no ggml MoE kernel for ", ggml_type_name(type), " with T=", t);+ ggml_cuda_pool_alloc<float> x32(ctx.pool());+ const Activation a = activation(x, mmvq || mmq_reads_16bit(type), x32, stream);+ ggml_tensor src1 = describe(a.type, a.data, {k, ky, t, 1}, {a.esz, a.esz * k, a.esz * k * ky, a.esz * k * ky * t});+ const size_t oe = ggml_type_size(otype);+ ggml_tensor dst = describe(otype, out.data_ptr(), {n, top_k, t, 1}, {oe, oe * n, oe * n * top_k, oe * n * top_k * t});+ if (mmvq) {+ ggml_cuda_mul_mat_vec_q(ctx, &src0, &src1, &tids, &dst);+ } else {+ ggml_cuda_mul_mat_q(ctx, &src0, &src1, &tids, &dst, 0);+ }+}++namespace {++// The GLUs the MMVQ epilogue computes; any other ggml_glu_op falls through to a plain gate * up there.+ggml_glu_op fused_glu_op(const String & name) {+ if (name == "swiglu") {+ return GGML_GLU_OP_SWIGLU;+ }+ if (name == "geglu") {+ return GGML_GLU_OP_GEGLU;+ }+ if (name == "swiglu_oai") {+ return GGML_GLU_OP_SWIGLU_OAI;+ }+ RuntimeCheck(name == "swiglu_clamp", "MMVQ fuses swiglu, geglu, swiglu_oai or swiglu_clamp, not ", std::string(name));+ return GGML_GLU_OP_SWIGLU_CLAMP;+}++} // namespace++// out[T, k, I] = glu(x @ gate^T, x @ up^T) for small T in one MMVQ pass; w[E, 2I, row_bytes] holds each expert's gate rows then its up rows; limit is swiglu_clamp's+void op_mul_mat_id_glu(TensorView w, int64_t type_id, TensorView x, TensorView ids, TensorView out, String glu, double limit) {+ const ggml_glu_op glu_op = fused_glu_op(glu);+ RuntimeCheck(w.ndim() == 3 && w.size(1) % 2 == 0, "mul_mat_id_glu takes w[E, 2I, row_bytes]");+ RuntimeCheck(is_dtype(ids.dtype(), kDLInt, 32) && ids.ndim() == 2 && ids.IsContiguous(), "ids must be a contiguous int32 [T, k] tensor");+ RuntimeCheck(x.ndim() == 2 && x.size(0) == ids.size(0), "x must be [T, K]");+ const DeviceGuard guard(x.device());+ const ggml_type type = (ggml_type) type_id;+ const int64_t t = ids.size(0), top_k = ids.size(1), k = x.size(1), e = w.size(0), n = w.size(1) / 2;+ check_weight(w, type, k);+ RuntimeCheck(out.IsContiguous() && out.ndim() == 3 && out.size(0) == t && out.size(1) == top_k && out.size(2) == n,+ "out must be a contiguous [T, k, I] tensor");+ const ggml_type otype = io_type(x, out);+ const int cc = device_cc();+ RuntimeCheck(t <= MMVQ_MAX_BATCH_SIZE && t <= get_mmvq_mmid_max_batch(type, cc), "mul_mat_id_glu is the small-batch path, T=", t);+ if (t == 0) {+ return;+ }+ auto & ctx = context(x.device());+ ggml_cuda_pool_alloc<float> x32(ctx.pool());+ const Activation a = activation(x, true, x32, ctx.stream());+ const size_t s1 = w.stride(1), s2 = w.stride(0);+ const char * base = (const char *) w.data_ptr();+ ggml_tensor up = describe(type, base + n * s1, {k, n, e, 1}, {ggml_type_size(type), s1, s2, s2 * e});+ ggml_tensor gate = describe(type, base, {k, n, e, 1}, {ggml_type_size(type), s1, s2, s2 * e});+ ggml_tensor src1 = describe(a.type, a.data, {k, 1, t, 1}, {a.esz, a.esz * k, a.esz * k, a.esz * k * t});+ const size_t ids_stride = sizeof(int32_t) * top_k, ids_bytes = ids_stride * t;+ ggml_tensor tids = describe(GGML_TYPE_I32, ids.data_ptr(), {top_k, t, 1, 1}, {sizeof(int32_t), ids_stride, ids_bytes, ids_bytes});+ const size_t oe = ggml_type_size(otype);+ ggml_tensor dst = describe(otype, out.data_ptr(), {n, top_k, t, 1}, {oe, oe * n, oe * n * top_k, oe * n * top_k * t});+ ggml_cuda_mm_fusion_args_host fusion{};+ fusion.gate = &gate;+ fusion.glu_op = glu_op;+ fusion.glu_limit = (float) limit;+ ggml_cuda_mul_mat_vec_q(ctx, &up, &src1, &tids, &dst, &fusion);+}++namespace {++template <typename T>+__global__ void moe_weighted_sum_kernel(const T * __restrict__ y, const float * __restrict__ w, T * __restrict__ out,+ const int64_t n, const int top_k) {+ const int64_t token = blockIdx.x;+ const int64_t col = (int64_t) blockIdx.y * blockDim.x + threadIdx.x;+ if (col >= n) {+ return;+ }+ const int64_t row0 = token * top_k;+ float sum = 0.0f;+ for (int j = 0; j < top_k; ++j) {+ sum += ggml_cuda_cast<float>(y[(row0 + j) * n + col]) * w[row0 + j];+ }+ out[token * n + col] = ggml_cuda_cast<T>(sum);+}++} // namespace++// out[T, N] = sum_j weights[t, j] * y[t, j, :], summed in fp32+void op_moe_weighted_sum(TensorView y, TensorView weights, TensorView out) {+ RuntimeCheck(y.ndim() == 3 && y.IsContiguous(), "y must be a contiguous [T, k, N] tensor");+ const int64_t t = y.size(0), top_k = y.size(1), n = y.size(2);+ RuntimeCheck(float_type(weights) == GGML_TYPE_F32 && weights.IsContiguous() && weights.numel() == t * top_k, "weights must be a contiguous fp32 [T, k] tensor");+ RuntimeCheck(out.ndim() == 2 && out.size(0) == t && out.size(1) == n && out.IsContiguous(), "out must be a contiguous [T, N] tensor");+ const ggml_type otype = io_type(y, out);+ const DeviceGuard guard(y.device());+ if (t == 0) {+ return;+ }+ constexpr int threads = 256;+ const dim3 blocks(t, (n + threads - 1) / threads);+ const cudaStream_t stream = current_stream(y.device());+ const float * wp = (const float *) weights.data_ptr();+ if (otype == GGML_TYPE_BF16) {+ moe_weighted_sum_kernel<<<blocks, threads, 0, stream>>>((const nv_bfloat16 *) y.data_ptr(), wp, (nv_bfloat16 *) out.data_ptr(), n, top_k);+ } else if (otype == GGML_TYPE_F16) {+ moe_weighted_sum_kernel<<<blocks, threads, 0, stream>>>((const half *) y.data_ptr(), wp, (half *) out.data_ptr(), n, top_k);+ } else {+ moe_weighted_sum_kernel<<<blocks, threads, 0, stream>>>((const float *) y.data_ptr(), wp, (float *) out.data_ptr(), n, top_k);+ }+ CUDA_CHECK(cudaGetLastError());+}++// out[rows, K] = W[rows, row_bytes] dequantized to out's dtype+void op_dequantize(TensorView w, int64_t type_id, int64_t k, TensorView out) {+ const DeviceGuard guard(w.device());+ const ggml_type type = (ggml_type) type_id;+ check_weight(w, type, k);+ RuntimeCheck(w.IsContiguous(), "dequantize needs a contiguous weight");+ const int64_t rows = w.numel() / w.size(w.ndim() - 1);+ RuntimeCheck(out.IsContiguous() && out.numel() == rows * k, "out must be a contiguous [rows, K] tensor");+ const ggml_type otype = float_type(out);+ if (rows == 0) {+ return;+ }+ const cudaStream_t stream = current_stream(w.device());+ if (type == otype) {+ CUDA_CHECK(cudaMemcpyAsync(out.data_ptr(), w.data_ptr(), rows * k * ggml_type_size(type), cudaMemcpyDeviceToDevice, stream));+ return;+ }+ if (otype == GGML_TYPE_BF16) {+ const auto convert = ggml_get_to_bf16_cuda(type);+ RuntimeCheck(convert != nullptr, "no ggml bf16 conversion for ", ggml_type_name(type));+ convert(w.data_ptr(), (nv_bfloat16 *) out.data_ptr(), rows * k, stream);+ } else if (otype == GGML_TYPE_F16) {+ const auto convert = ggml_get_to_fp16_cuda(type);+ RuntimeCheck(convert != nullptr, "no ggml fp16 conversion for ", ggml_type_name(type));+ convert(w.data_ptr(), (half *) out.data_ptr(), rows * k, stream);+ } else {+ const auto convert = ggml_get_to_fp32_cuda(type);+ RuntimeCheck(convert != nullptr, "no ggml fp32 conversion for ", ggml_type_name(type));+ convert(w.data_ptr(), (float *) out.data_ptr(), rows * k, stream);+ }+}++// out[i] = dequant(W[ids[i]]) for a table W[rows, row_bytes]+void op_get_rows(TensorView w, int64_t type_id, int64_t k, TensorView ids, TensorView out) {+ RuntimeCheck(is_dtype(ids.dtype(), kDLInt, 32) && ids.ndim() == 1 && ids.IsContiguous(), "ids must be a contiguous int32 vector");+ RuntimeCheck(w.ndim() == 2, "get_rows takes w[rows, row_bytes]");+ const DeviceGuard guard(w.device());+ const ggml_type type = (ggml_type) type_id;+ check_weight(w, type, k);+ const int64_t n = ids.size(0);+ RuntimeCheck(out.IsContiguous() && out.ndim() == 2 && out.size(0) == n && out.size(1) == k, "out must be a contiguous [n, K] tensor");+ const ggml_type otype = float_type(out);+ if (n == 0) {+ return;+ }+ const size_t nb01 = w.stride(0), esz = otype == GGML_TYPE_F32 ? 4 : 2;+ get_rows_cuda(w.data_ptr(), type, (const int32_t *) ids.data_ptr(), out.data_ptr(), otype, k, nb01, nb01 * w.size(0), nb01 * w.size(0),+ n, 1, 1, 4, 4 * n, 4 * n, k * esz, k * esz * n, k * esz * n, current_stream(w.device()));+}++bool op_uses_mmvq(int64_t type_id, int64_t m) { return ggml_cuda_should_use_mmvq((ggml_type) type_id, device_cc(), m); }++int64_t op_mmvq_max_tokens(int64_t type_id) {+ return std::min<int64_t>(MMVQ_MAX_BATCH_SIZE, get_mmvq_mmid_max_batch((ggml_type) type_id, device_cc()));+}++TVM_FFI_DLL_EXPORT_TYPED_FUNC(quantize_q8_1, op_quantize_q8_1);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(q8_1_row_bytes, q8_1_row_bytes);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mmq_tail_bytes, mmq_tail_bytes);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mul_mat, op_mul_mat);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mul_mat_id, op_mul_mat_id);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mul_mat_id_glu, op_mul_mat_id_glu);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(moe_weighted_sum, op_moe_weighted_sum);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(dequantize, op_dequantize);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(get_rows, op_get_rows);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(uses_mmvq, op_uses_mmvq);+TVM_FFI_DLL_EXPORT_TYPED_FUNC(mmvq_max_tokens, op_mmvq_max_tokens);diff --git a/python/freetoken/kernel/csrc/ggml/freetoken/ggml_shim.cu b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_shim.cunew file mode 100644index 00000000..80ef7285--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/freetoken/ggml_shim.cu@@ -0,0 +1,255 @@+// The ggml-base and ggml-cuda.cu host symbols that the vendored ggml-cuda kernels reference.+// Tensor descriptors built by ggml_ops.cu never have a backend buffer, and temporary buffers come from tvm-ffi's env allocator (torch's caching allocator under torch).++#include "common.cuh"++#include <tvm/ffi/container/tensor.h>+#include <tvm/ffi/extra/c_env_api.h>++#include <cstdarg>+#include <cstdio>+#include <stdexcept>+#include <string>+#include <unordered_map>++namespace {++struct TypeInfo {+ const char * name = nullptr;+ int64_t blck_size = 0;+ size_t type_size = 0;+ bool quantized = false;+};++const TypeInfo * type_infos() {+ static TypeInfo t[GGML_TYPE_COUNT];+ static const bool init = [] {+ auto set = [](ggml_type type, const char * name, int64_t blck, size_t size, bool quantized) {+ t[type] = {name, blck, size, quantized};+ };+ set(GGML_TYPE_F32, "f32", 1, sizeof(float), false);+ set(GGML_TYPE_F16, "f16", 1, sizeof(ggml_fp16_t), false);+ set(GGML_TYPE_BF16, "bf16", 1, sizeof(ggml_bf16_t), false);+ set(GGML_TYPE_F64, "f64", 1, sizeof(double), false);+ set(GGML_TYPE_I8, "i8", 1, sizeof(int8_t), false);+ set(GGML_TYPE_I16, "i16", 1, sizeof(int16_t), false);+ set(GGML_TYPE_I32, "i32", 1, sizeof(int32_t), false);+ set(GGML_TYPE_I64, "i64", 1, sizeof(int64_t), false);+ set(GGML_TYPE_Q1_0, "q1_0", QK1_0, sizeof(block_q1_0), true);+ set(GGML_TYPE_Q2_0, "q2_0", QK2_0, sizeof(block_q2_0), true);+ set(GGML_TYPE_Q4_0, "q4_0", QK4_0, sizeof(block_q4_0), true);+ set(GGML_TYPE_Q4_1, "q4_1", QK4_1, sizeof(block_q4_1), true);+ set(GGML_TYPE_Q5_0, "q5_0", QK5_0, sizeof(block_q5_0), true);+ set(GGML_TYPE_Q5_1, "q5_1", QK5_1, sizeof(block_q5_1), true);+ set(GGML_TYPE_Q8_0, "q8_0", QK8_0, sizeof(block_q8_0), true);+ set(GGML_TYPE_Q8_1, "q8_1", QK8_1, sizeof(block_q8_1), true);+ set(GGML_TYPE_MXFP4, "mxfp4", QK_MXFP4, sizeof(block_mxfp4), true);+ set(GGML_TYPE_NVFP4, "nvfp4", QK_NVFP4, sizeof(block_nvfp4), true);+ set(GGML_TYPE_TQ1_0, "tq1_0", QK_K, sizeof(block_tq1_0), true);+ set(GGML_TYPE_TQ2_0, "tq2_0", QK_K, sizeof(block_tq2_0), true);+ set(GGML_TYPE_Q2_K, "q2_K", QK_K, sizeof(block_q2_K), true);+ set(GGML_TYPE_Q3_K, "q3_K", QK_K, sizeof(block_q3_K), true);+ set(GGML_TYPE_Q4_K, "q4_K", QK_K, sizeof(block_q4_K), true);+ set(GGML_TYPE_Q5_K, "q5_K", QK_K, sizeof(block_q5_K), true);+ set(GGML_TYPE_Q6_K, "q6_K", QK_K, sizeof(block_q6_K), true);+ set(GGML_TYPE_Q8_K, "q8_K", QK_K, sizeof(block_q8_K), true);+ set(GGML_TYPE_IQ2_XXS, "iq2_xxs", QK_K, sizeof(block_iq2_xxs), true);+ set(GGML_TYPE_IQ2_XS, "iq2_xs", QK_K, sizeof(block_iq2_xs), true);+ set(GGML_TYPE_IQ2_S, "iq2_s", QK_K, sizeof(block_iq2_s), true);+ set(GGML_TYPE_IQ3_XXS, "iq3_xxs", QK_K, sizeof(block_iq3_xxs), true);+ set(GGML_TYPE_IQ3_S, "iq3_s", QK_K, sizeof(block_iq3_s), true);+ set(GGML_TYPE_IQ1_S, "iq1_s", QK_K, sizeof(block_iq1_s), true);+ set(GGML_TYPE_IQ1_M, "iq1_m", QK_K, sizeof(block_iq1_m), true);+ set(GGML_TYPE_IQ4_NL, "iq4_nl", QK4_NL, sizeof(block_iq4_nl), true);+ set(GGML_TYPE_IQ4_XS, "iq4_xs", QK_K, sizeof(block_iq4_xs), true);+ return true;+ }();+ (void) init;+ return t;+}++const TypeInfo & type_info(ggml_type type) {+ if (type < 0 || type >= GGML_TYPE_COUNT || type_infos()[type].name == nullptr) {+ throw std::invalid_argument("unknown ggml type " + std::to_string((int) type));+ }+ return type_infos()[type];+}++struct EnvPool : ggml_cuda_pool {+ explicit EnvPool(int device) : device(device) {}++ void * alloc(size_t size, size_t * actual_size) override {+ *actual_size = size;+ if (size == 0) {+ return nullptr;+ }+#ifdef GGML_USE_HIP+ constexpr DLDeviceType device_type = kDLROCM;+#else+ constexpr DLDeviceType device_type = kDLCUDA;+#endif+ auto buf = tvm::ffi::Tensor::FromEnvAlloc(TVMFFIEnvTensorAlloc, tvm::ffi::Shape({(int64_t) size}), DLDataType{kDLUInt, 8, 1},+ DLDevice{device_type, device});+ void * ptr = buf.data_ptr();+ live.emplace(ptr, std::move(buf));+ return ptr;+ }++ void free(void * ptr, size_t) override { live.erase(ptr); }++ int device;+ std::unordered_map<void *, tvm::ffi::Tensor> live;+};++} // namespace++void ggml_abort(const char * file, int line, const char * fmt, ...) {+ char msg[1024];+ va_list args;+ va_start(args, fmt);+ vsnprintf(msg, sizeof(msg), fmt, args);+ va_end(args);+ throw std::runtime_error(std::string("ggml: ") + file + ":" + std::to_string(line) + ": " + msg);+}++void ggml_log_internal(enum ggml_log_level level, const char * format, ...) {+ if (level < GGML_LOG_LEVEL_WARN || level == GGML_LOG_LEVEL_CONT) {+ return;+ }+ va_list args;+ va_start(args, format);+ vfprintf(stderr, format, args);+ va_end(args);+}++int64_t ggml_blck_size(enum ggml_type type) { return type_info(type).blck_size; }++size_t ggml_type_size(enum ggml_type type) { return type_info(type).type_size; }++const char * ggml_type_name(enum ggml_type type) { return type_info(type).name; }++bool ggml_is_quantized(enum ggml_type type) { return type_info(type).quantized; }++size_t ggml_row_size(enum ggml_type type, int64_t ne) { return ggml_type_size(type) * ne / ggml_blck_size(type); }++size_t ggml_element_size(const struct ggml_tensor * tensor) { return ggml_type_size(tensor->type); }++int64_t ggml_nelements(const struct ggml_tensor * tensor) {+ return tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3];+}++size_t ggml_nbytes(const struct ggml_tensor * tensor) {+ for (int i = 0; i < GGML_MAX_DIMS; ++i) {+ if (tensor->ne[i] <= 0) {+ return 0;+ }+ }+ size_t nbytes;+ const size_t blck_size = ggml_blck_size(tensor->type);+ if (blck_size == 1) {+ nbytes = ggml_type_size(tensor->type);+ for (int i = 0; i < GGML_MAX_DIMS; ++i) {+ nbytes += (tensor->ne[i] - 1) * tensor->nb[i];+ }+ } else {+ nbytes = tensor->ne[0] * tensor->nb[0] / blck_size;+ for (int i = 1; i < GGML_MAX_DIMS; ++i) {+ nbytes += (tensor->ne[i] - 1) * tensor->nb[i];+ }+ }+ return nbytes;+}++bool ggml_is_matrix(const struct ggml_tensor * tensor) { return tensor->ne[2] == 1 && tensor->ne[3] == 1; }++bool ggml_is_contiguous(const struct ggml_tensor * tensor) {+ size_t next_nb = ggml_type_size(tensor->type);+ if (tensor->ne[0] != ggml_blck_size(tensor->type) && tensor->nb[0] != next_nb) {+ return false;+ }+ next_nb *= tensor->ne[0] / ggml_blck_size(tensor->type);+ for (int i = 1; i < GGML_MAX_DIMS; i++) {+ if (tensor->ne[i] != 1 && tensor->nb[i] != next_nb) {+ return false;+ }+ next_nb *= tensor->ne[i];+ }+ return true;+}++bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor) {+ return ggml_nbytes(tensor) == ggml_nelements(tensor) * ggml_type_size(tensor->type) / ggml_blck_size(tensor->type);+}++bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {+ return t0->ne[0] == t1->ne[0] && t0->ne[1] == t1->ne[1] && t0->ne[2] == t1->ne[2] && t0->ne[3] == t1->ne[3];+}++bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {+ return t0->nb[0] == t1->nb[0] && t0->nb[1] == t1->nb[1] && t0->nb[2] == t1->nb[2] && t0->nb[3] == t1->nb[3];+}++enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage(ggml_backend_buffer_t) {+ return GGML_BACKEND_BUFFER_USAGE_WEIGHTS;+}++size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t, const struct ggml_tensor * tensor) {+ return ggml_nbytes(tensor);+}++[[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) {+ throw std::runtime_error(std::string("ggml-cuda: ") + msg + " in " + func + " at " + file + ":" + std::to_string(line) + ": " + stmt);+}++int ggml_cuda_get_device() {+ int id;+ CUDA_CHECK(cudaGetDevice(&id));+ return id;+}++void ggml_cuda_set_device(int device) {+ int current;+ CUDA_CHECK(cudaGetDevice(¤t));+ if (current != device) {+ CUDA_CHECK(cudaSetDevice(device));+ }+}++const ggml_cuda_device_info & ggml_cuda_info() {+ static const ggml_cuda_device_info info = [] {+ ggml_cuda_device_info in = {};+ CUDA_CHECK(cudaGetDeviceCount(&in.physical_device_count));+ GGML_ASSERT(in.physical_device_count <= GGML_CUDA_MAX_DEVICES);+ in.device_count = in.physical_device_count;+ for (int id = 0; id < in.device_count; ++id) {+ cudaDeviceProp prop;+ CUDA_CHECK(cudaGetDeviceProperties(&prop, id));+ auto & d = in.devices[id];+#ifdef GGML_USE_HIP+ GGML_ASSERT(std::string(prop.gcnArchName).starts_with("gfx"));+ d.cc = GGML_CUDA_CC_OFFSET_AMD + static_cast<int>(std::strtol(prop.gcnArchName + 3, nullptr, 16));+#else+ d.cc = 100 * prop.major + 10 * prop.minor;+#endif+ d.nsm = prop.multiProcessorCount;+ d.smpb = prop.sharedMemPerBlock;+ d.smpbo = prop.sharedMemPerBlockOptin;+ d.warp_size = prop.warpSize;+ d.total_vram = prop.totalGlobalMem;+ d.physical_device = id;+ d.physical_share_count = 1;+ int coop = 0;+ CUDA_CHECK(cudaDeviceGetAttribute(&coop, cudaDevAttrCooperativeLaunch, id));+ d.supports_cooperative_launch = coop != 0;+ }+ return in;+ }();+ return info;+}++// The streams belong to the caller, so there is nothing to release.+ggml_backend_cuda_context::~ggml_backend_cuda_context() {}++std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(int device, int) {+ return std::make_unique<EnvPool>(device);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuhnew file mode 100644index 00000000..36912ce4--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/common.cuh@@ -0,0 +1,1722 @@+#pragma once++#include "ggml.h"+#include "ggml-impl.h"+#include "ggml-cuda.h"++#include <cstdint>+#include <cstdlib>+#include <memory>+#include <mutex>++#if defined(GGML_USE_HIP)+#define GGML_COMMON_DECL_HIP+#define GGML_COMMON_IMPL_HIP+#else+#define GGML_COMMON_DECL_CUDA+#define GGML_COMMON_IMPL_CUDA+#if defined(GGML_USE_MUSA)+#define GGML_COMMON_DECL_MUSA+#define GGML_COMMON_IMPL_MUSA+#endif+#endif+#include "ggml-common.h"++#include <array>+#include <algorithm>+#include <cassert>+#include <cfloat>+#include <cstdio>+#include <string>+#include <unordered_map>+#include <utility>+#include <vector>++#if defined(GGML_USE_HIP)+#include "vendors/hip.h"+#elif defined(GGML_USE_MUSA)+#include "vendors/musa.h"+#else+#include "vendors/cuda.h"+#endif // defined(GGML_USE_HIP)++#define STRINGIZE_IMPL(...) #__VA_ARGS__+#define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__)++#define WARP_SIZE 32+#define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed)+#define CUDART_HMASK 12000 // CUDA 12.0, min. ver. for half2 -> uint mask comparisons++#define GGML_CUDA_CC_PASCAL 600+#define GGML_CUDA_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products+#define GGML_CUDA_CC_VOLTA 700+#define GGML_CUDA_CC_TURING 750+#define GGML_CUDA_CC_AMPERE 800+#define GGML_CUDA_CC_ORIN 870+#define GGML_CUDA_CC_ADA_LOVELACE 890+#define GGML_CUDA_CC_HOPPER 900+// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see+// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms+#define GGML_CUDA_CC_BLACKWELL 1200+#define GGML_CUDA_CC_DGX_SPARK 1210+#define GGML_CUDA_CC_RUBIN 1300+#define GGML_CUDA_CC_OFFSET_AMD 0x1000000+#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000+#define GGML_CUDA_CC_IS_NVIDIA(cc) (cc < GGML_CUDA_CC_OFFSET_MTHREADS)++// AMD+// GCN/CDNA, wave size is 64+#define GGML_CUDA_CC_GCN4 (GGML_CUDA_CC_OFFSET_AMD + 0x803) // Tonga, Fiji, Polaris, minimum for fast fp16+#define GGML_CUDA_CC_VEGA (GGML_CUDA_CC_OFFSET_AMD + 0x900) // Vega56/64, minimum for fp16 dual issue+#define GGML_CUDA_CC_VEGA20 (GGML_CUDA_CC_OFFSET_AMD + 0x906) // MI50/Radeon VII, minimum for dp4a+#define GGML_CUDA_CC_GFX909 (GGML_CUDA_CC_OFFSET_AMD + 0x909) // GCN APU+#define GGML_CUDA_CC_GFX90C (GGML_CUDA_CC_OFFSET_AMD + 0x90c) // GCN APU+#define GGML_CUDA_CC_CDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x908) // MI100, minimum for MFMA, acc registers+#define GGML_CUDA_CC_CDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x90a) // MI210 (gfx90a), minimum acc register renaming+#define GGML_CUDA_CC_CDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x942) // MI300+#define GGML_CUDA_CC_CDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x950) // MI350X/MI355X++// RDNA removes MFMA, dp4a, xnack, acc registers, wave size is 32+#define GGML_CUDA_CC_RDNA1 (GGML_CUDA_CC_OFFSET_AMD + 0x1010) // RX 5000+#define GGML_CUDA_CC_RDNA2 (GGML_CUDA_CC_OFFSET_AMD + 0x1030) // RX 6000, minimum for dp4a+#define GGML_CUDA_CC_RDNA3 (GGML_CUDA_CC_OFFSET_AMD + 0x1100) // RX 7000, minimum for WMMA+#define GGML_CUDA_CC_RDNA3_5 (GGML_CUDA_CC_OFFSET_AMD + 0x1150) // AI 370, AI Max 395 laptops.+#define GGML_CUDA_CC_RDNA4 (GGML_CUDA_CC_OFFSET_AMD + 0x1200) // RX 9000++#define GGML_CUDA_CC_IS_AMD(cc) (cc >= GGML_CUDA_CC_OFFSET_AMD)+#define GGML_CUDA_CC_IS_RDNA(cc) (cc >= GGML_CUDA_CC_RDNA1)+#define GGML_CUDA_CC_IS_RDNA1(cc) (cc >= GGML_CUDA_CC_RDNA1 && cc < GGML_CUDA_CC_RDNA2)+#define GGML_CUDA_CC_IS_RDNA2(cc) (cc >= GGML_CUDA_CC_RDNA2 && cc < GGML_CUDA_CC_RDNA3)+#define GGML_CUDA_CC_IS_RDNA3_0(cc) (cc >= GGML_CUDA_CC_RDNA3 && cc < GGML_CUDA_CC_RDNA3_5)+#define GGML_CUDA_CC_IS_RDNA3_5(cc) (cc >= GGML_CUDA_CC_RDNA3_5 && cc < GGML_CUDA_CC_RDNA4)+#define GGML_CUDA_CC_IS_RDNA3(cc) (GGML_CUDA_CC_IS_RDNA3_0(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc))+#define GGML_CUDA_CC_IS_RDNA4(cc) (cc >= GGML_CUDA_CC_RDNA4)+#define GGML_CUDA_CC_IS_GCN_APU(cc) ((cc) == GGML_CUDA_CC_GFX909 || (cc) == GGML_CUDA_CC_GFX90C)+#define GGML_CUDA_CC_IS_GCN(cc) ((cc > GGML_CUDA_CC_OFFSET_AMD && cc < GGML_CUDA_CC_CDNA1) || GGML_CUDA_CC_IS_GCN_APU(cc))+#define GGML_CUDA_CC_IS_CDNA(cc) (!GGML_CUDA_CC_IS_GCN_APU(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_RDNA1)+#define GGML_CUDA_CC_IS_CDNA1(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA1 && cc < GGML_CUDA_CC_CDNA2)+#define GGML_CUDA_CC_IS_CDNA2(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA2 && cc < GGML_CUDA_CC_CDNA3)+#define GGML_CUDA_CC_IS_CDNA3(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA3 && cc < GGML_CUDA_CC_CDNA4)+#define GGML_CUDA_CC_IS_CDNA4(cc) (GGML_CUDA_CC_IS_CDNA(cc) && cc >= GGML_CUDA_CC_CDNA4 && cc < GGML_CUDA_CC_RDNA1)++// Moore Threads+#define MUSART_HMASK 40300 // MUSA rc4.3, min. ver. for half2 -> uint mask comparisons++#define GGML_CUDA_CC_QY1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x210) // MTT S80, MTT S3000+#define GGML_CUDA_CC_QY2 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x220) // MTT S4000+#define GGML_CUDA_CC_PH1 (GGML_CUDA_CC_OFFSET_MTHREADS + 0x310) // MTT S5000++#define GGML_CUDA_CC_IS_MTHREADS(cc) (cc >= GGML_CUDA_CC_OFFSET_MTHREADS && cc < GGML_CUDA_CC_OFFSET_AMD)+#define GGML_CUDA_CC_IS_QY1(cc) (cc >= GGML_CUDA_CC_QY1 && cc < GGML_CUDA_CC_QY2)+#define GGML_CUDA_CC_IS_QY2(cc) (cc >= GGML_CUDA_CC_QY2 && cc < GGML_CUDA_CC_PH1)+#define GGML_CUDA_CC_IS_PH1(cc) (cc >= GGML_CUDA_CC_PH1)++#if !defined(GGML_USE_HIP) && (defined(GGML_USE_MUSA) || CUDART_VERSION >= 11070)+# define GGML_CUDA_USE_CUB+#endif // !defined(GGML_USE_HIP) && (defined(GGML_USE_MUSA) || CUDART_VERSION >= 11070)++// PDL host-side support (cudaLaunchKernelEx) requires CUDART >= 11.8.+// However, this has been bugged in CTK < 12.3 for MSVC builds, see+// https://github.com/ggml-org/llama.cpp/pull/22522#discussion_r3302393293+// __CUDA_ARCH__ is undefined in host passes; GPU arch check happens in device-side code.+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && \+ (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080))+# define GGML_CUDA_USE_PDL+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080))++static __device__ __forceinline__ void ggml_cuda_syncwarp() {+#ifndef GGML_USE_HIP+ __syncwarp();+#endif // GGML_USE_HIP+}++static __device__ __forceinline__ void ggml_cuda_pdl_sync() {+#if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER+ cudaGridDependencySynchronize();+#endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER+}++static __device__ __forceinline__ void ggml_cuda_pdl_lc() {+#if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER+ cudaTriggerProgrammaticLaunchCompletion();+#endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER+}++#ifdef __CUDA_ARCH_LIST__+constexpr bool ggml_cuda_has_arch_impl(int) {+ return false;+}++template<class ... Archs>+constexpr bool ggml_cuda_has_arch_impl(const int arch, const int first, Archs... rest) {+ return arch == first || ggml_cuda_has_arch_impl(arch, rest...);+}++constexpr bool ggml_cuda_has_arch(const int arch) {+ return ggml_cuda_has_arch_impl(arch, __CUDA_ARCH_LIST__);+}++constexpr int ggml_cuda_highest_compiled_arch_impl(const int /*arch*/, const int cur) {+ if (cur == 0) {+ return -1;+ }+ return cur;+}++template<class ... Archs>+constexpr int ggml_cuda_highest_compiled_arch_impl(const int arch, const int cur, const int first, Archs... rest) {+ if (first <= arch && first > cur) {+ return ggml_cuda_highest_compiled_arch_impl(arch, first, rest...);+ } else {+ return ggml_cuda_highest_compiled_arch_impl(arch, cur, rest...);+ }+}++constexpr int ggml_cuda_highest_compiled_arch(const int arch) {+ return ggml_cuda_highest_compiled_arch_impl(arch, 0, __CUDA_ARCH_LIST__);+}+#else+static int ggml_cuda_highest_compiled_arch(const int arch) {+ return arch;+}+#endif // __CUDA_ARCH_LIST__++// ---------------------------------------------------------------------------------------------------------++#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses++#define GGML_CUDA_MAX_STREAMS 8++[[noreturn]]+void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg);++#define CUDA_CHECK_GEN(err, success, error_fn) \+ do { \+ auto err_ = (err); \+ if (err_ != (success)) { \+ ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_)); \+ } \+ } while (0)++#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString)+++#if CUDART_VERSION >= 12000 || defined(GGML_USE_MUSA)+ static const char * cublas_get_error_str(const cublasStatus_t err) {+ return cublasGetStatusString(err);+ }+#else+ static const char * cublas_get_error_str(const cublasStatus_t err) {+ switch (err) {+ case CUBLAS_STATUS_SUCCESS: return "CUBLAS_STATUS_SUCCESS";+ case CUBLAS_STATUS_NOT_INITIALIZED: return "CUBLAS_STATUS_NOT_INITIALIZED";+ case CUBLAS_STATUS_ALLOC_FAILED: return "CUBLAS_STATUS_ALLOC_FAILED";+ case CUBLAS_STATUS_INVALID_VALUE: return "CUBLAS_STATUS_INVALID_VALUE";+ case CUBLAS_STATUS_ARCH_MISMATCH: return "CUBLAS_STATUS_ARCH_MISMATCH";+ case CUBLAS_STATUS_MAPPING_ERROR: return "CUBLAS_STATUS_MAPPING_ERROR";+ case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED";+ case CUBLAS_STATUS_INTERNAL_ERROR: return "CUBLAS_STATUS_INTERNAL_ERROR";+ case CUBLAS_STATUS_NOT_SUPPORTED: return "CUBLAS_STATUS_NOT_SUPPORTED";+ default: return "unknown error";+ }+ }+#endif // CUDART_VERSION >= 12000++#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str)++#ifdef GGML_USE_NCCL+#define NCCL_CHECK(err) CUDA_CHECK_GEN(err, ncclSuccess, ncclGetErrorString)+#endif // GGML_USE_NCCL++#if !defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)+static const char * cu_get_error_str(CUresult err) {+ const char * err_str;+ cuGetErrorString(err, &err_str);+ return err_str;+}+#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str)+#endif++#if !defined(GGML_USE_HIP)+# define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \+ do { \+ static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = { false }; \+ const int id = ggml_cuda_get_device(); \+ if (!shared_memory_limit_raised[id]) { \+ CUDA_CHECK(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes)); \+ shared_memory_limit_raised[id] = true; \+ } \+ } while (0)+#else+# define CUDA_SET_SHARED_MEMORY_LIMIT(kernel, nbytes) \+ do { \+ GGML_UNUSED(nbytes); \+ } while (0)+#endif // !defined(GGML_USE_HIP)++#if CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)+#define GGML_CUDA_ASSUME(x) __builtin_assume(x)+#else+#define GGML_CUDA_ASSUME(x)+#endif // CUDART_VERSION >= 11010++#if (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM))+#define GGML_USE_VMM+#endif // (!defined(GGML_USE_HIP) && !defined(GGML_CUDA_NO_VMM)) || (defined(GGML_USE_HIP) && !defined(GGML_HIP_NO_VMM))++#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL+#define FP16_AVAILABLE+#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) || __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL++#if defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610+#define FAST_FP16_AVAILABLE+#endif // defined(FP16_AVAILABLE) && __CUDA_ARCH__ != 610++#if defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)+#define AMD_MFMA_AVAILABLE+#endif // defined(GGML_USE_HIP) && defined(CDNA) && !defined(GGML_HIP_NO_MMQ_MFMA)++#if defined(GGML_USE_HIP) && (defined(RDNA4) || defined(RDNA3))+#define AMD_WMMA_AVAILABLE+#endif // defined(GGML_USE_HIP) && defined(RDNA4)++// The Volta instructions are in principle available on Turing or newer but they are effectively unusable:+#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA+#define VOLTA_MMA_AVAILABLE+#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA++#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING+#define TURING_MMA_AVAILABLE+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING++#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+#define AMPERE_MMA_AVAILABLE+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE++// FreeToken: the native fp4 MMA needs an sm_120a/sm_120f build, so it is off unless GGML_CUDA_FP4_MMA is defined; FP4 weights then take Q8_1 activations+#if !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL && __CUDA_ARCH__ < GGML_CUDA_CC_RUBIN && defined(GGML_CUDA_FP4_MMA)+# define BLACKWELL_MMA_AVAILABLE+#endif // !defined(GGML_USE_HIP) && __CUDA_ARCH__ >= GGML_CUDA_CC_BLACKWELL++#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+#define CP_ASYNC_AVAILABLE+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE++#if !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220)+#define FLASH_ATTN_AVAILABLE+#endif // !defined(GGML_CUDA_NO_FA) && !(defined(GGML_USE_MUSA) && __MUSA_ARCH__ < 220)++static bool fp16_available(const int cc) {+ return ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_PASCAL ||+ (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1);+}++static bool fast_fp16_available(const int cc) {+ return GGML_CUDA_CC_IS_AMD(cc) ||+ (GGML_CUDA_CC_IS_NVIDIA(cc) && fp16_available(cc) && ggml_cuda_highest_compiled_arch(cc) != 610) ||+ (GGML_CUDA_CC_IS_MTHREADS(cc) && fp16_available(cc));+}++// To be used for feature selection of external libraries, e.g. cuBLAS.+static bool fast_fp16_hardware_available(const int cc) {+ return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_PASCAL && cc != 610) || GGML_CUDA_CC_IS_AMD(cc) ||+ (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);+}++// To be used for feature selection of external libraries, e.g. cuBLAS.+static bool fp16_mma_hardware_available(const int cc) {+ return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_VOLTA) ||+ GGML_CUDA_CC_IS_CDNA(cc) || GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc) ||+ (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);+}++// To be used for feature selection of external libraries, e.g. cuBLAS.+static bool fast_bf16_hardware_available(const int cc) {+ return (GGML_CUDA_CC_IS_AMD(cc) && (cc >= GGML_CUDA_CC_RDNA3 || GGML_CUDA_CC_IS_CDNA(cc)))+ || (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE);+}++static bool bf16_mma_hardware_available(const int cc) {+ return (GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_AMPERE) ||+ GGML_CUDA_CC_IS_CDNA(cc) || cc >= GGML_CUDA_CC_RDNA3 ||+ (GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_PH1);+}++static bool fp32_mma_hardware_available(const int cc) {+ return GGML_CUDA_CC_IS_CDNA(cc);+}++static bool amd_mfma_available(const int cc) {+#if !defined(GGML_HIP_NO_MMQ_MFMA)+ return GGML_CUDA_CC_IS_CDNA(cc);+#else+ return false;+#endif //!defined(GGML_HIP_NO_MMQ_MFMA)+}++static bool amd_wmma_available(const int cc) {+ return (GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_RDNA3(cc));+}++static bool volta_mma_available(const int cc) {+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_VOLTA;+}++static bool turing_mma_available(const int cc) {+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_TURING;+}++static bool ampere_mma_available(const int cc) {+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;+}++static bool cp_async_available(const int cc) {+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_AMPERE;+}++static bool blackwell_mma_available(const int cc) {+#ifdef GGML_CUDA_FP4_MMA+ return GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_BLACKWELL &&+ ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;+#else+ GGML_UNUSED(cc);+ return false;+#endif // GGML_CUDA_FP4_MMA+}++// Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes.+static bool ggml_cuda_is_aligned(const ggml_tensor * tensor, const size_t alignment) {+ GGML_ASSERT(tensor != nullptr);+ return (reinterpret_cast<uintptr_t>(tensor->data) % alignment) == 0 &&+ tensor->nb[1] % alignment == 0 &&+ tensor->nb[2] % alignment == 0 &&+ tensor->nb[3] % alignment == 0;+}++static constexpr __device__ int ggml_cuda_get_physical_warp_size() {+#if defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))+ return 64;+#else+ return 32;+#endif // defined(GGML_USE_HIP) && (defined(__GFX9__) || defined(__GFX8__))+}++// Maximum number of bytes that can be copied in a single instruction.+static constexpr __device__ int ggml_cuda_get_max_cpy_bytes() {+#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)+ return 16;+#else+#if __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA+ return 16;+#else+ return 8;+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA+#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)+}+++[[noreturn]]+static __device__ void no_device_code(+ const char * file_name, const int line, const char * function_name, const int arch, const char * arch_list) {++#if defined(GGML_USE_HIP)+ printf("%s:%d: ERROR: HIP kernel %s has no device code compatible with HIP arch %d.\n",+ file_name, line, function_name, arch);+ GGML_UNUSED(arch_list);+#else+ printf("%s:%d: ERROR: CUDA kernel %s has no device code compatible with CUDA arch %d. ggml-cuda.cu was compiled for: %s\n",+ file_name, line, function_name, arch, arch_list);+#endif // defined(GGML_USE_HIP)+ __trap();++ GGML_UNUSED(no_device_code); // suppress unused function warning+}++#ifdef __CUDA_ARCH__+#define NO_DEVICE_CODE no_device_code(__FILE__, __LINE__, __FUNCTION__, __CUDA_ARCH__, STRINGIZE(__CUDA_ARCH_LIST__))+#else+#define NO_DEVICE_CODE //GGML_ABORT("NO_DEVICE_CODE not valid in host code.")+#endif // __CUDA_ARCH__++// The compiler is always able to unroll loops if they contain continue expressions.+// In such cases loop unrolling can still be achieved via recursion:+template <int n>+struct ggml_cuda_unroll {+ template <typename Func, typename... Args>+ __device__ void operator()(const Func & f, Args... args) const {+ f(n - 1, args...);+ ggml_cuda_unroll<n - 1>{}(f, args...);+ }+};++template <>+struct ggml_cuda_unroll<1> {+ template <typename Func, typename... Args>+ __device__ void operator()(const Func & f, Args... args) const {+ f(0, args...);+ }+};++template<int width = WARP_SIZE>+static __device__ __forceinline__ int warp_reduce_sum(int x) {+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+ return __reduce_add_sync(0xffffffff, x);+#else+#pragma unroll+ for (int offset = width/2; offset > 0; offset >>= 1) {+ x += __shfl_xor_sync(0xffffffff, x, offset, width);+ }+ return x;+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ float warp_reduce_sum(float x) {+#pragma unroll+ for (int offset = width/2; offset > 0; offset >>= 1) {+ x += __shfl_xor_sync(0xffffffff, x, offset, width);+ }+ return x;+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {+#pragma unroll+ for (int offset = width/2; offset > 0; offset >>= 1) {+ a.x += __shfl_xor_sync(0xffffffff, a.x, offset, width);+ a.y += __shfl_xor_sync(0xffffffff, a.y, offset, width);+ }+ return a;+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {+#ifdef FP16_AVAILABLE+#pragma unroll+ for (int offset = width/2; offset > 0; offset >>= 1) {+ a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, offset, width));+ }+ return a;++#else+ NO_DEVICE_CODE;+ return a;+#endif // FP16_AVAILABLE+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ int warp_reduce_all(int x) {+ if (width == ggml_cuda_get_physical_warp_size()) {+ return __all_sync(0xffffffff, x);+ } else {+#pragma unroll+ for (int offset = width/2; offset > 0; offset >>= 1) {+ x = __shfl_xor_sync(0xffffffff, x, offset, width) && x;+ }+ return x;+ }+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ int warp_reduce_any(int x) {+ if (width == ggml_cuda_get_physical_warp_size()) {+ return __any_sync(0xffffffff, x);+ } else {+#pragma unroll+ for (int offset = width/2; offset > 0; offset >>= 1) {+ x = __shfl_xor_sync(0xffffffff, x, offset, width) || x;+ }+ return x;+ }+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ float warp_reduce_max(float x) {+#pragma unroll+ for (int offset = width/2; offset > 0; offset >>= 1) {+ x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, offset, width));+ }+ return x;+}++template<typename T, int width = WARP_SIZE>+static __device__ __forceinline__ T warp_prefix_inclusive_sum(T x) {+ const int lane_id = threadIdx.x % width;+#pragma unroll+ for (int offset = 1; offset < width; offset <<= 1) {+ const T t = __shfl_up_sync(0xffffffff, x, offset, width);+ if (lane_id >= offset) {+ x += t;+ }+ }+ return x;+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ float2 warp_prefix_inclusive_sum(float2 a) {+ const int lane_id = threadIdx.x % width;+#pragma unroll+ for (int offset = 1; offset < width; offset <<= 1) {+ const float t_x = __shfl_up_sync(0xffffffff, a.x, offset, width);+ const float t_y = __shfl_up_sync(0xffffffff, a.y, offset, width);+ if (lane_id >= offset) {+ a.x += t_x;+ a.y += t_y;+ }+ }+ return a;+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ half2 warp_prefix_inclusive_sum(half2 a) {+#ifdef FP16_AVAILABLE+ const int lane_id = threadIdx.x % width;+#pragma unroll+ for (int offset = 1; offset < width; offset <<= 1) {+ const half2 t = __shfl_up_sync(0xffffffff, a, offset, width);+ if (lane_id >= offset) {+ a = __hadd2(a, t);+ }+ }+ return a;++#else+ NO_DEVICE_CODE;+ return a;+#endif // FP16_AVAILABLE+}++enum class block_reduce_method {+ MAX,+ SUM,+};++template<block_reduce_method method_t, typename T>+struct block_reduce_policy;++template <typename T, typename... Ts>+inline constexpr bool is_any = (std::is_same_v<T, Ts> || ...);++template<typename...>+inline constexpr bool ggml_cuda_dependent_false_v = false;++template <typename T> struct block_reduce_policy<block_reduce_method::SUM, T> {+ static __device__ T reduce(T val) {+ if constexpr(is_any<T, float, float2, half2, int>) {+ return warp_reduce_sum(val);+ } else {+ static_assert(ggml_cuda_dependent_false_v<T>, "Unsupported type for block reduce sum");+ }+ }++ static __device__ T sentinel() {+ if constexpr (std::is_same_v<T, float>) {+ return 0.0f;+ } else if constexpr (std::is_same_v<T, float2>) {+ return make_float2(0.0f, 0.0f);+ } else if constexpr (std::is_same_v<T, half2>) {+ return make_half2(0.0f, 0.0f);+ } else if constexpr (std::is_same_v<T, int>) {+ return 0;+ } else {+ static_assert(ggml_cuda_dependent_false_v<T>, "Unsupported type for block reduce sum");+ }+ }+};++template <typename T> struct block_reduce_policy<block_reduce_method::MAX, T> {+ static __device__ T reduce(T val) {+ if constexpr (is_any<T, float, half2>) {+ return warp_reduce_max(val);+ } else {+ static_assert(ggml_cuda_dependent_false_v<T>, "Unsupported type for block reduce max");+ }+ }++ static __device__ T sentinel() {+ if constexpr (std::is_same_v<T, float>) {+ return -INFINITY;+ } else if constexpr (std::is_same_v<T, half2>) {+ return make_half2(-INFINITY, -INFINITY);+ } else {+ static_assert(ggml_cuda_dependent_false_v<T>, "Unsupported type for block reduce max");+ }+ }+};++template <block_reduce_method reduce_method_t, const unsigned int block_size_template = 0, typename T>+static __device__ T block_reduce(T val, [[maybe_unused]] T * shared_vals) {+ // for multi-warp reductions, callers must not reuse shared_vals until all reads from this invocation have completed+ val = block_reduce_policy<reduce_method_t, T>::reduce(val);+ const unsigned int block_size = block_size_template == 0 ? blockDim.x : block_size_template;+ if (block_size > WARP_SIZE) {+ assert((block_size <= 1024) && (block_size % WARP_SIZE) == 0);+ const int warp_id = threadIdx.x / WARP_SIZE;+ const int lane_id = threadIdx.x % WARP_SIZE;+ if (lane_id == 0) {+ shared_vals[warp_id] = val;+ }+ __syncthreads();+ val = block_reduce_policy<reduce_method_t, T>::sentinel();+ if (lane_id < (static_cast<int>(block_size) / WARP_SIZE)) {+ val = shared_vals[lane_id];+ }+ return block_reduce_policy<reduce_method_t, T>::reduce(val);+ }++ return val;+}++static __device__ __forceinline__ half ggml_cuda_hmax(const half a, const half b) {+#ifdef FP16_AVAILABLE++#if !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX+ return __float2half(fmaxf(__half2float(a), __half2float(b)));+#else+ return __hmax(a, b);+#endif // !defined(GGML_USE_HIP) && CUDART_VERSION < CUDART_HMAX++#else+ NO_DEVICE_CODE;+ GGML_UNUSED(b);+ return a;+#endif // FP16_AVAILABLE+}++static __device__ __forceinline__ half2 ggml_cuda_hmax2(const half2 a, const half2 b) {+#if defined(GGML_USE_HIP)+ return half2(__hmax(a.x, b.x), __hmax(a.y, b.y));+#elif CUDART_VERSION >= CUDART_HMAX+ return __hmax2(a, b);+#else+ half2 ret;+ reinterpret_cast<half&>(ret.x) = __float2half(fmaxf( __low2float(a), __low2float(b)));+ reinterpret_cast<half&>(ret.y) = __float2half(fmaxf(__high2float(a), __high2float(b)));+ return ret;+#endif+}++template<int width = WARP_SIZE>+static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {+#pragma unroll+ for (int offset = width/2; offset > 0; offset >>= 1) {+ x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, offset, width));+ }+ return x;+}++#if (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || \+ (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK)+static __device__ __forceinline__ uint32_t __hgt2_mask(const half2 a, const half2 b) {+ const uint32_t mask_low = 0x0000FFFF * (float( __low2half(a)) > float( __low2half(b)));+ const uint32_t mask_high = 0xFFFF0000 * (float(__high2half(a)) > float(__high2half(b)));+ return mask_low | mask_high;+}+#endif // (defined(CUDART_VERSION) && CUDART_VERSION < CUDART_HMASK) || defined(GGML_USE_HIP) || (defined(MUSART_VERSION) && MUSART_VERSION < MUSART_HMASK)++static __device__ __forceinline__ int ggml_cuda_dp4a(const int a, const int b, int c) {+#if defined(GGML_USE_HIP)+#if defined(CDNA) || defined(RDNA2) || defined(__gfx906__)+ c = __builtin_amdgcn_sdot4(a, b, c, false);+#elif defined(RDNA3) || defined(RDNA4)+ c = __builtin_amdgcn_sudot4( true, a, true, b, c, false);+#elif defined(RDNA1) || defined(__gfx900__)+ int tmp1;+ int tmp2;+ asm("\n \+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0 \n \+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 \n \+ v_add3_u32 %0, %1, %2, %0 \n \+ v_mul_i32_i24 %1, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2 \n \+ v_mul_i32_i24 %2, sext(%3), sext(%4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3 \n \+ v_add3_u32 %0, %1, %2, %0 \n \+ "+ : "+v"(c), "=&v"(tmp1), "=&v"(tmp2)+ : "v"(a), "v"(b)+ );+#else+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);+ c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];+#endif+ return c;++#else // defined(GGML_USE_HIP)++#if __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)+ return __dp4a(a, b, c);+#else // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)+ const int8_t * a8 = (const int8_t *) &a;+ const int8_t * b8 = (const int8_t *) &b;+ return c + a8[0]*b8[0] + a8[1]*b8[1] + a8[2]*b8[2] + a8[3]*b8[3];+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A || defined(GGML_USE_MUSA)++#endif // defined(GGML_USE_HIP)+}++static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float v, const float u) {+ acc += v*u;+}++static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const float2 v, const float2 u) {+ acc += v.x*u.x;+ acc += v.y*u.y;+}++#if defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))+#define V_DOT2_F32_F16_AVAILABLE+#endif // defined(GGML_USE_HIP) && (defined(RDNA2) || defined(RDNA3) || defined(RDNA4) || defined(__gfx906__) || defined(CDNA))++static __device__ __forceinline__ void ggml_cuda_mad(float & acc, const half2 v, const half2 u) {+#ifdef V_DOT2_F32_F16_AVAILABLE+ asm volatile("v_dot2_f32_f16 %0, %1, %2, %0" : "+v"(acc) : "v"(v), "v"(u));+#else+#ifdef FAST_FP16_AVAILABLE+ const float2 tmp = __half22float2(v*u);+ acc += tmp.x + tmp.y;+#else+ const float2 tmpv = __half22float2(v);+ const float2 tmpu = __half22float2(u);+ acc += tmpv.x * tmpu.x;+ acc += tmpv.y * tmpu.y;+#endif // FAST_FP16_AVAILABLE+#endif // V_DOT2_F32_F16_AVAILABLE+}++static __device__ __forceinline__ void ggml_cuda_mad(half2 & acc, const half2 v, const half2 u) {+#ifdef FAST_FP16_AVAILABLE+ acc += v*u;+#else+ const float2 tmpv = __half22float2(v);+ const float2 tmpu = __half22float2(u);+ float2 tmpacc = __half22float2(acc);+ tmpacc.x += tmpv.x * tmpu.x;+ tmpacc.y += tmpv.y * tmpu.y;+ acc = make_half2(tmpacc.x, tmpacc.y);+#endif // FAST_FP16_AVAILABLE+}++// Aligned memory transfers of 8/16 bytes can be faster than 2 transfers with 4 bytes, especially on AMD.+// Important: do not use this function if dst and src both point at registers.+// Due to the strict aliasing rule the compiler can do incorrect optimizations if src and dst have different types.+// The function is intended for copies between registers and SRAM/VRAM to make the compiler emit the right instructions.+// If dst and src point at different address spaces then they are guaranteed to not be aliased.+template <int nbytes, int alignment = 0>+static __device__ __forceinline__ void ggml_cuda_memcpy_1(void * __restrict__ dst, const void * __restrict__ src) {+ static_assert(+ nbytes <= ggml_cuda_get_max_cpy_bytes() || alignment == 0,+ "You are misusing the alignment parameter for ggml_cuda_memcpy_1. "+ "The intent is for the parameter is only as a workaround if either one of the pointers is not properly aligned. "+ "If you use it to do more bytes per copy than ggml_cuda_max_cpy_bytes() the reads and writes may not be coalesced. "+ "Call ggml_cuda_memcpy_1 in a loop instead.");+ if constexpr (alignment != 0) {+ static_assert(nbytes % alignment == 0, "bad alignment");+ }+ constexpr int nb_per_cpy = alignment == 0 ? nbytes : alignment;++#pragma unroll+ for (int i = 0; i < nbytes/nb_per_cpy; ++i) {+ if constexpr (nb_per_cpy == 1) {+ ((char *) dst)[i] = ((const char *) src)[i];+ } else if constexpr (nb_per_cpy == 2) {+ ((short *) dst)[i] = ((const short *) src)[i];+ } else if constexpr (nb_per_cpy == 4) {+ ((int *) dst)[i] = ((const int *) src)[i];+ } else if constexpr (nb_per_cpy == 8) {+ ((int2 *) dst)[i] = ((const int2 *) src)[i];+ } else if constexpr (nb_per_cpy == 16) {+ ((int4 *) dst)[i] = ((const int4 *) src)[i];+ } else {+ static_assert(nbytes == 0 && nbytes == -1, "bad nbytes");+ }+ }+}++static __device__ __forceinline__ float ggml_cuda_e8m0_to_fp32(uint8_t x) {+#if CUDART_VERSION >= 12080+ const nv_bfloat16 e = __nv_cvt_e8m0_to_bf16raw(x);+ return (float) e;+#else+ uint32_t bits;+ if (x == 0) {+ bits = 0x00400000;+ } else {+ bits = (uint32_t) x << 23;+ }++ float result;+ memcpy(&result, &bits, sizeof(float));+ return result;+#endif // CUDART_VERSION >= 12050+}++static __device__ __forceinline__ float ggml_cuda_ue4m3_to_fp32(uint8_t x) {+#if defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000+ // ROCm does not support fp8 in software on devices with fp8 hardware,+ // but CDNA3 supports only e4m3_fnuz (no inf).+ const uint32_t bits = x * (x != 0x7F && x != 0xFF); // Convert NaN to 0.0f to match CPU implementation.+ const __hip_fp8_e4m3_fnuz xf = *reinterpret_cast<const __hip_fp8_e4m3_fnuz *>(&bits);+ return static_cast<float>(xf) / 2;+#else+#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP)+ const uint32_t bits = x * (x != 0x7F && x != 0xFF); // Convert NaN to 0.0f to match CPU implementation.+ const __nv_fp8_e4m3 xf = *reinterpret_cast<const __nv_fp8_e4m3 *>(&bits);+ return static_cast<float>(xf) / 2;+#else+ if (x == 0 || (x == 0x7F && x != 0xFF)) { // Convert NaN to 0.0f+ return 0.0f;+ }+ const int exp = (x >> 3) & 0xF;+ const int man = x & 0x7;+ float raw;+ if (exp == 0) {+ raw = ldexpf((float) man, -9);+ } else {+ raw = ldexpf(1.0f + (float) man / 8.0f, exp - 7);+ }+ return static_cast<float>(raw / 2);+#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP)+#endif // defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000+}++static __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_ue4m3(float x) {+#if defined(BLACKWELL_MMA_AVAILABLE) // This is used for NVFP4 subblock scale quantizations only+ if (!(x > 0.0f)) {+ return 0;+ }+ const __nv_fp8_e4m3 xf(x);+ return xf.__x;+#else+ GGML_UNUSED(x);+ NO_DEVICE_CODE; // Used only for NVFP4 Scales for Activations, only for Blackwell+#endif // defined(BLACKWELL_MMA_AVAILABLE)+}++__device__ __forceinline__ uint8_t ggml_cuda_float_to_fp4_e2m1(float x, float e) {+ const uint8_t sign_bit = (x < 0.0f) << 3;+ float ax = fabsf(x) * e;++ // Positive LUT+ static constexpr float pos_lut[8] = { 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f };++ int best_i = 0;+ float best_err = fabsf(ax - pos_lut[0]);++#pragma unroll+ for (int i = 1; i < 8; ++i) {+ const float err = fabsf(ax - pos_lut[i]);+ if (err < best_err) {+ best_err = err;+ best_i = i;+ }+ }++ return static_cast<uint8_t>(best_i | sign_bit);+}++// See https://gmplib.org/~tege/divcnst-pldi94.pdf figure 4.1.+// Precompute mp (m' in the paper) and L such that division+// can be computed using a multiply (high 32b of 64b result)+// and a shift:+//+// n/d = (mulhi(n, mp) + n) >> L;+static const uint3 init_fastdiv_values(uint64_t d_64) {+ GGML_ASSERT(d_64 != 0);+ GGML_ASSERT(d_64 <= std::numeric_limits<uint32_t>::max());++ uint32_t d = (uint32_t)d_64;++ // compute L = ceil(log2(d));+ uint32_t L = 0;+ while (L < 32 && (uint32_t{ 1 } << L) < d) {+ L++;+ }++ uint32_t mp = (uint32_t) ((uint64_t{ 1 } << 32) * ((uint64_t{ 1 } << L) - d) / d + 1);+ // pack divisor as well to reduce error surface+ return make_uint3(mp, L, d);+}++static __device__ __forceinline__ uint32_t fastdiv(uint32_t n, const uint3 fastdiv_values) {+ // expects fastdiv_values to contain <mp, L, divisor> in <x, y, z>+ // fastdiv_values.z is unused and optimized away by the compiler.+ // Compute high 32 bits of n * mp+ const uint32_t hi = __umulhi(n, fastdiv_values.x);+ // add n, apply bit shift+ return (hi + n) >> fastdiv_values.y;+}++static __device__ __forceinline__ uint32_t fastmodulo(uint32_t n, const uint3 fastdiv_values) {+ // expects fastdiv_values to contain <mp, L, divisor> in <x, y, z> (see init_fastdiv_values)+ return n - fastdiv(n, fastdiv_values) * fastdiv_values.z;+}++// Calculate both division and modulo at once, returns <n/divisor, n%divisor>+static __device__ __forceinline__ uint2 fast_div_modulo(uint32_t n, const uint3 fastdiv_values) {+ // expects fastdiv_values to contain <mp, L, divisor> in <x, y, z> (see init_fastdiv_values)+ const uint32_t div_val = fastdiv(n, fastdiv_values);+ const uint32_t mod_val = n - div_val * fastdiv_values.z;+ return make_uint2(div_val, mod_val);+}++typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, float2 & v);++template<typename dst_t>+using dequantize_kq_t = void (*)(const void * vx, const int64_t ib, dst_t * y, const int tid);++static __device__ __forceinline__ float get_alibi_slope(+ const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1+) {+ if (max_bias <= 0.0f) {+ return 1.0f;+ }+ const float base = h < n_head_log2 ? m0 : m1;+ const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1;++ return powf(base, exph);+}++template <ggml_type type>+struct ggml_cuda_type_traits;++template<>+struct ggml_cuda_type_traits<GGML_TYPE_F16> {+ static constexpr int qk = 1;+ static constexpr int qr = 1;+ static constexpr int bs = sizeof(ggml_half);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q1_0> {+ static constexpr int qk = QK1_0;+ static constexpr int qr = QR1_0;+ static constexpr int qi = QI1_0;+ static constexpr int bs = sizeof(block_q1_0);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q2_0> {+ static constexpr int qk = QK2_0;+ static constexpr int qr = QR2_0;+ static constexpr int qi = QI2_0;+ static constexpr int bs = sizeof(block_q2_0);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q4_0> {+ static constexpr int qk = QK4_0;+ static constexpr int qr = QR4_0;+ static constexpr int qi = QI4_0;+ static constexpr int bs = sizeof(block_q4_0);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q4_1> {+ static constexpr int qk = QK4_1;+ static constexpr int qr = QR4_1;+ static constexpr int qi = QI4_1;+ static constexpr int bs = sizeof(block_q4_1);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q5_0> {+ static constexpr int qk = QK5_0;+ static constexpr int qr = QR5_0;+ static constexpr int qi = QI5_0;+ static constexpr int bs = sizeof(block_q5_0);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q5_1> {+ static constexpr int qk = QK5_1;+ static constexpr int qr = QR5_1;+ static constexpr int qi = QI5_1;+ static constexpr int bs = sizeof(block_q5_1);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q8_0> {+ static constexpr int qk = QK8_0;+ static constexpr int qr = QR8_0;+ static constexpr int qi = QI8_0;+ static constexpr int bs = sizeof(block_q8_0);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_MXFP4> {+ static constexpr int qk = QK_MXFP4;+ static constexpr int qr = QR_MXFP4;+ static constexpr int qi = QI_MXFP4;+ static constexpr int bs = sizeof(block_mxfp4);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_NVFP4> {+ static constexpr int qk = QK_NVFP4;+ static constexpr int qr = QR_NVFP4;+ static constexpr int qi = QI_NVFP4;+ static constexpr int bs = sizeof(block_nvfp4);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q2_K> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR2_K;+ static constexpr int qi = QI2_K;+ static constexpr int bs = sizeof(block_q2_K);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q3_K> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR3_K;+ static constexpr int qi = QI3_K;+ static constexpr int bs = sizeof(block_q3_K);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q4_K> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR4_K;+ static constexpr int qi = QI4_K;+ static constexpr int bs = sizeof(block_q4_K);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q5_K> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR5_K;+ static constexpr int qi = QI5_K;+ static constexpr int bs = sizeof(block_q5_K);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_Q6_K> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR6_K;+ static constexpr int qi = QI6_K;+ static constexpr int bs = sizeof(block_q6_K);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XXS> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR2_XXS;+ static constexpr int qi = QI2_XXS;+ static constexpr int bs = sizeof(block_iq2_xxs);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_XS> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR2_XS;+ static constexpr int qi = QI2_XS;+ static constexpr int bs = sizeof(block_iq2_xs);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ2_S> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR2_S;+ static constexpr int qi = QI2_S;+ static constexpr int bs = sizeof(block_iq2_s);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ3_XXS> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR3_XXS;+ static constexpr int qi = QI3_XXS;+ static constexpr int bs = sizeof(block_iq3_xxs);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ1_S> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR1_S;+ static constexpr int qi = QI1_S;+ static constexpr int bs = sizeof(block_iq1_s);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ1_M> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR1_M;+ static constexpr int qi = QI1_M;+ static constexpr int bs = sizeof(block_iq1_m);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ4_NL> {+ static constexpr int qk = QK4_NL;+ static constexpr int qr = QR4_NL;+ static constexpr int qi = QI4_NL;+ static constexpr int bs = sizeof(block_iq4_nl);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ4_XS> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR4_XS;+ static constexpr int qi = QI4_XS;+ static constexpr int bs = sizeof(block_iq4_xs);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_IQ3_S> {+ static constexpr int qk = QK_K;+ static constexpr int qr = QR3_S;+ static constexpr int qi = QI3_S;+ static constexpr int bs = sizeof(block_iq3_s);+};++//////////////////////++struct ggml_cuda_device_info {+ int device_count; // number of (possibly virtual) devices exposed to the rest of ggml+ int physical_device_count; // number of physical CUDA devices actually present++ struct cuda_device_info {+ int cc; // compute capability+ int nsm; // number of streaming multiprocessors+ size_t smpb; // max. shared memory per block+ size_t smpbo; // max. shared memory per block (with opt-in)+ bool integrated; // Device is integrated as opposed to discrete+ bool vmm; // virtual memory support+ size_t vmm_granularity; // granularity of virtual memory+ size_t total_vram;+ int warp_size; // Number of threads in a dispatch+ bool supports_cooperative_launch; // whether cooperative launch is supported+ int physical_device; // backing physical CUDA device for this (virtual) device+ int physical_share_count; // number of (virtual) devices sharing this device's physical GPU+ int virtual_index; // index of this (virtual) device among those sharing its physical GPU+ };++ cuda_device_info devices[GGML_CUDA_MAX_DEVICES] = {};++ std::array<float, GGML_CUDA_MAX_DEVICES> default_tensor_split = {};+};++const ggml_cuda_device_info & ggml_cuda_info();++void ggml_cuda_set_device(int device);+int ggml_cuda_get_device();++struct ggml_cuda_pool {+ virtual ~ggml_cuda_pool() = default;++ virtual void * alloc(size_t size, size_t * actual_size) = 0;+ virtual void free(void * ptr, size_t size) = 0;+};++template<typename T>+struct ggml_cuda_pool_alloc {+ ggml_cuda_pool * pool = nullptr;+ T * ptr = nullptr;+ size_t actual_size = 0;++ ggml_cuda_pool_alloc() = default;++ explicit ggml_cuda_pool_alloc(ggml_cuda_pool & pool) : pool(&pool) {+ }++ ggml_cuda_pool_alloc(ggml_cuda_pool & pool, size_t size) : pool(&pool) {+ alloc(size);+ }++ ~ggml_cuda_pool_alloc() {+ if (ptr != nullptr) {+ pool->free(ptr, actual_size);+ }+ }++ // size is in number of elements+ T * alloc(size_t size) {+ GGML_ASSERT(pool != nullptr);+ GGML_ASSERT(ptr == nullptr);+ ptr = (T *) pool->alloc(size * sizeof(T), &this->actual_size);+ return ptr;+ }++ T * alloc(ggml_cuda_pool & pool, size_t size) {+ this->pool = &pool;+ return alloc(size);+ }++ T * get() {+ return ptr;+ }++ ggml_cuda_pool_alloc(const ggml_cuda_pool_alloc &) = delete;+ ggml_cuda_pool_alloc(ggml_cuda_pool_alloc &&) = delete;+ ggml_cuda_pool_alloc& operator=(const ggml_cuda_pool_alloc &) = delete;+ ggml_cuda_pool_alloc& operator=(ggml_cuda_pool_alloc &&) = delete;+};+++// backend interface++struct ggml_tensor_extra_gpu {+ void * data_device[GGML_CUDA_MAX_DEVICES]; // 1 pointer for each device for split tensors+ cudaEvent_t events[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS]; // events for synchronizing multiple GPUs+};+++#if (defined(GGML_CUDA_USE_GRAPHS) || defined(GGML_HIP_GRAPHS)) || defined(GGML_MUSA_GRAPHS)+#define USE_CUDA_GRAPH+#endif++struct ggml_cuda_graph {+#ifdef USE_CUDA_GRAPH+ ~ggml_cuda_graph() {+ if (instance != nullptr) {+ CUDA_CHECK(cudaGraphExecDestroy(instance));+ }+ if (graph != nullptr) {+ CUDA_CHECK(cudaGraphDestroy(graph));+ }+ }+ cudaGraph_t graph = nullptr;+ cudaGraphExec_t instance = nullptr;+ size_t num_nodes = 0;+ std::vector<cudaGraphNode_t> nodes;+ bool disable_due_to_gpu_arch = false;+ bool warmup_complete = false;+ uint64_t uid = 0;+ int64_t last_used_time = 0;+ struct node_properties {+ ggml_tensor node;+ void * node_src_data_ptrs[GGML_MAX_SRC];+ int64_t node_src_ne[GGML_MAX_SRC][GGML_MAX_DIMS];+ size_t node_src_nb[GGML_MAX_SRC][GGML_MAX_DIMS];+ };+ std::vector<node_properties> node_props;++ bool is_enabled() const {+ static const bool disable_cuda_graphs_due_to_env = (getenv("GGML_CUDA_DISABLE_GRAPHS") != nullptr);+ return !(disable_due_to_gpu_arch || disable_cuda_graphs_due_to_env);+ }+#endif+};++struct ggml_cuda_concurrent_event {+ std::vector<cudaEvent_t> join_events;+ cudaEvent_t fork_event = nullptr;++ int n_streams = 0;+ std::unordered_map<const ggml_tensor *, int> stream_mapping;++ // Original order of nodes in this concurrent region (before interleaving)+ // Used to restore grouping for fusion within streams+ std::vector<const ggml_tensor *> original_order;++ const ggml_tensor * join_node;++ ggml_cuda_concurrent_event() = default;++ ggml_cuda_concurrent_event(const ggml_cuda_concurrent_event &) = delete;+ ggml_cuda_concurrent_event & operator=(const ggml_cuda_concurrent_event &) = delete;++ explicit ggml_cuda_concurrent_event(int n_streams) : n_streams(n_streams) {+ join_events.resize(n_streams);++ for (size_t i = 0; i < join_events.size(); ++i) {+ CUDA_CHECK(cudaEventCreateWithFlags(&join_events[i], cudaEventDisableTiming));+ }++ CUDA_CHECK(cudaEventCreateWithFlags(&fork_event, cudaEventDisableTiming));+ }++ ggml_cuda_concurrent_event(ggml_cuda_concurrent_event && other) noexcept+ : join_events(std::move(other.join_events))+ , fork_event(other.fork_event)+ , n_streams(other.n_streams)+ , stream_mapping(std::move(other.stream_mapping))+ , original_order(std::move(other.original_order))+ , join_node(other.join_node) {+ other.fork_event = nullptr;+ }++ // 1. check if any branches write to overlapping memory ranges (except the join node)+ // 2. check whether all srcs are either within the branch or outside the nodes covered by ggml_cuda_concurrent_event+ // we assume all nodes have the same buffer+ bool is_valid() const {+ std::vector<std::vector<std::pair<int64_t, int64_t>>> write_ranges;+ write_ranges.resize(n_streams);++ // get join_node's memory range to exclude from overlap checking.+ // multiple nodes can use join_node's buffer; we synchronize on the join node.+ const ggml_tensor * join_t = join_node->view_src ? join_node->view_src : join_node;+ const int64_t join_start = (int64_t) join_t->data;+ const int64_t join_end = join_start + ggml_nbytes(join_t);++ for (const auto & [tensor, stream] : stream_mapping) {+ const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor;+ const int64_t t_start = (int64_t) t->data;+ const int64_t t_end = t_start + ggml_nbytes(t);++ // skip tensors that overlap with join_node's buffer.+ if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) {+ continue;+ }++ // concurrent streams begin from 1+ write_ranges[stream - 1].emplace_back(t_start, t_end);+ }++ for (int i = 0; i < n_streams; ++i) {+ // sorts first by start then by end of write range+ std::sort(write_ranges[i].begin(), write_ranges[i].end());+ }++ bool writes_overlap = false;+ bool dependent_srcs = false;+ for (const auto & [tensor, stream] : stream_mapping) {+ const ggml_tensor * t = tensor->view_src ? tensor->view_src : tensor;+ const int64_t t_start = (int64_t) t->data;+ const int64_t t_end = t_start + ggml_nbytes(t);++ // skip tensors that overlap with join_node's buffer+ if ((t_start <= join_start && join_start < t_end) || (join_start <= t_start && t_start < join_end)) {+ continue;+ }++ // check if this buffer's write data overlaps with another stream's+ std::pair<int64_t, int64_t> data_range = std::make_pair(t_start, t_end);+ for (int i = 0; i < n_streams; ++i) {+ if (i == stream - 1) {+ continue;+ }+ auto it = std::lower_bound(write_ranges[i].begin(), write_ranges[i].end(), data_range);++ if (it != write_ranges[i].end()) {+ const std::pair<int64_t, int64_t> & other = *it;++ // std::lower_bound returns the first element where other >= data_range (lexicographically).+ // This guarantees other.first >= data_range.first.+ // Therefore, overlap occurs iff other.first < data_range.second+ // (i.e., the other range starts before this range ends).+ if (other.first < data_range.second) {+ GGML_LOG_DEBUG("Writes overlap for %s", tensor->name);+ writes_overlap = true;+ break;+ }+ }+ }++ //check if all srcs are either in branch or don't have a branch+ for (int i = 0; i < GGML_MAX_SRC; ++i) {+ if (!tensor->src[i]) {+ continue;+ }++ auto it = stream_mapping.find(tensor->src[i]);++ if (it == stream_mapping.end()) {+ continue;+ }++ if (it->second != stream) {+ dependent_srcs = true;+ break;+ }+ }++ if (dependent_srcs || writes_overlap) {+ break;+ }+ }++ return !writes_overlap && !dependent_srcs;+ }++ ~ggml_cuda_concurrent_event() {+ if (fork_event != nullptr) {+ CUDA_CHECK(cudaEventDestroy(fork_event));+ }+ for (cudaEvent_t e : join_events) {+ if (e != nullptr) {+ CUDA_CHECK(cudaEventDestroy(e));+ }+ }+ }+};++struct ggml_cuda_stream_context {+ std::unordered_map<const ggml_tensor *, ggml_cuda_concurrent_event> concurrent_events;++ void reset() {+ concurrent_events.clear();+ }+};++struct ggml_backend_cuda_context {+ int device;+ std::string name;+ cudaEvent_t copy_event = nullptr;++ cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};+ cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};+ void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};+ size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {};++ int curr_stream_no = 0;++#ifdef USE_CUDA_GRAPH+ // Map from first_node_ptr to cuda_graph - allows multiple graphs per context+ // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe)+ std::unordered_map<const void *, std::unique_ptr<ggml_cuda_graph>> cuda_graphs;++ int64_t last_graph_eviction_sweep = 0;++ ggml_cuda_graph * cuda_graph(const void * first_node_ptr) {+ const int64_t time_now = ggml_time_us();++ // sweep every 5s, evicting cuda graphs unused for >=10s+ if (time_now - last_graph_eviction_sweep >= 5'000'000) {+ last_graph_eviction_sweep = time_now;+ for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) {+ if (time_now - it->second->last_used_time >= 10'000'000) {+ it = cuda_graphs.erase(it);+ } else {+ ++it;+ }+ }+ }++ auto it = cuda_graphs.find(first_node_ptr);+ if (it == cuda_graphs.end()) {+ it = cuda_graphs.emplace(first_node_ptr, std::make_unique<ggml_cuda_graph>()).first;+ }+ it->second->last_used_time = time_now;+ return it->second.get();+ }++ // Check if any CUDA graph is enabled for this context (used by kernels that need to know+ // if graphs are in use without having access to the specific graph key)+ bool any_cuda_graph_enabled() const {+ for (const auto & [key, graph] : cuda_graphs) {+ if (graph && graph->is_enabled()) {+ return true;+ }+ }+ return false;+ }++ // Check if any CUDA graph has an instance for this context+ bool any_cuda_graph_has_instance() const {+ for (const auto & [key, graph] : cuda_graphs) {+ if (graph && graph->instance != nullptr) {+ return true;+ }+ }+ return false;+ }+#endif // USE_CUDA_GRAPH++ explicit ggml_backend_cuda_context(int device) :+ device(device),+ name(GGML_CUDA_NAME + std::to_string(device)) {+ }++ ggml_cuda_stream_context concurrent_stream_context;++ ~ggml_backend_cuda_context();++ cudaStream_t stream(int device, int stream) {+ if (streams[device][stream] == nullptr) {+ ggml_cuda_set_device(device);+ CUDA_CHECK(cudaStreamCreateWithFlags(&streams[device][stream], cudaStreamNonBlocking));+ }+ return streams[device][stream];+ }++ cudaStream_t stream() { return stream(device, curr_stream_no); }++ ggml_cuda_stream_context & stream_context() { return concurrent_stream_context; }++ cublasHandle_t cublas_handle() {+ if (cublas_handles[device][curr_stream_no] == nullptr) {+ ggml_cuda_set_device(device);+ CUBLAS_CHECK(cublasCreate(&cublas_handles[device][curr_stream_no]));+ CUBLAS_CHECK(cublasSetMathMode(cublas_handles[device][curr_stream_no], CUBLAS_TF32_TENSOR_OP_MATH));+ CUBLAS_CHECK(cublasSetStream(cublas_handles[device][curr_stream_no], stream()));+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUBLAS_VER_MAJOR > 11 || (CUBLAS_VER_MAJOR == 11 && CUBLAS_VER_MINOR >= 2))+ if (cublas_workspace_sizes[device] == 0) {+ const int cc = ggml_cuda_info().devices[device].cc;+ cublas_workspace_sizes[device] = (cc >= GGML_CUDA_CC_HOPPER) ? 32 * 1024 * 1024 : 4 * 1024 * 1024;+ }+ CUDA_CHECK(cudaMalloc(&cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device]));+ CUBLAS_CHECK(cublasSetWorkspace(cublas_handles[device][curr_stream_no], cublas_workspaces[device][curr_stream_no], cublas_workspace_sizes[device]));+#endif+ }+ return cublas_handles[device][curr_stream_no];+ }++ // pool+ std::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS];++ static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(int device, int stream_no);++ ggml_cuda_pool & pool(int device) {+ if (pools[device][curr_stream_no] == nullptr) {+ pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no);+ }+ return *pools[device][curr_stream_no];+ }++ ggml_cuda_pool & pool() {+ return pool(device);+ }+};++struct ggml_cuda_mm_fusion_args_host {+ const ggml_tensor * x_bias = nullptr;+ const ggml_tensor * gate = nullptr;+ const ggml_tensor * gate_bias = nullptr;+ const ggml_tensor * x_scale = nullptr;+ const ggml_tensor * gate_scale = nullptr;+ ggml_glu_op glu_op;+ float glu_limit = 0.0f;+ const ggml_tensor * shared_up = nullptr;+ const ggml_tensor * shared_gate = nullptr;+ ggml_tensor * shared_dst = nullptr;+};+struct ggml_cuda_mm_fusion_args_device {+ const void * x_bias = nullptr;+ const void * gate = nullptr;+ const void * gate_bias = nullptr;+ const void * x_scale = nullptr;+ const void * gate_scale = nullptr;+ ggml_glu_op glu_op;+ float glu_limit = 0.0f;+ const void * shared_up = nullptr;+ const void * shared_gate = nullptr;+ float * shared_dst = nullptr;+ uint32_t shared_stride_col_dst = 0;+};++struct ggml_cuda_kernel_launch_params {+ dim3 block_nums;+ dim3 block_dims;+ size_t shmem;+ cudaStream_t stream;++ // size_t shmem+ ggml_cuda_kernel_launch_params(const dim3& block_nums_, const dim3& block_dims_, const size_t shmem_, const cudaStream_t stream_)+ : block_nums(block_nums_), block_dims(block_dims_), shmem(shmem_), stream(stream_) {}++ // Some call sites pass ints instead of the required size_t. This 2nd constructor casts int->size_t to avoid these -Wnarrowing warnings.+ ggml_cuda_kernel_launch_params(const dim3& block_nums_, const dim3& block_dims_, const int shmem_, const cudaStream_t stream_)+ : block_nums(block_nums_), block_dims(block_dims_), shmem((size_t)shmem_), stream(stream_) {}+};++#if defined(GGML_CUDA_USE_PDL)+struct ggml_cuda_pdl_config {+ cudaLaunchAttribute attr;+ cudaLaunchConfig_t cfg;++ ggml_cuda_pdl_config(const ggml_cuda_kernel_launch_params & params) {+ attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;+ attr.val.programmaticStreamSerializationAllowed = 1;++ cfg = {};+ cfg.gridDim = params.block_nums;+ cfg.blockDim = params.block_dims;+ cfg.dynamicSmemBytes = params.shmem;+ cfg.stream = params.stream;+ cfg.attrs = &attr;+ cfg.numAttrs = 1;+ }++ // Delete due to &attr+ ggml_cuda_pdl_config(const ggml_cuda_pdl_config&) = delete;+ ggml_cuda_pdl_config& operator=(const ggml_cuda_pdl_config&) = delete;+ ggml_cuda_pdl_config& operator=(ggml_cuda_pdl_config&&) = delete;++};++static bool ggml_cuda_kernel_can_use_pdl(const void * kernel) {+ const int device = ggml_cuda_get_device();++ struct cache_key {+ int device;+ const void * kernel;++ bool operator==(const cache_key & other) const { return device == other.device && kernel == other.kernel; }+ };++ struct cache_key_hash {+ // MurmurHash3 mixing function for better hash distribution (vs. just std::hash which in some implementations simply returns the identity)+ static size_t hash_mix(size_t x) {+ std::uint64_t y = x;+ const std::uint64_t m = 0xe9846af9b1a615d;++ y ^= y >> 32;+ y *= m;+ y ^= y >> 32;+ y *= m;+ y ^= y >> 28;++ return static_cast<size_t>(y);+ }++ size_t operator()(const cache_key & key) const {+ // Use a nonzero seed to avoid mapping all-zero keys to zero+ size_t h = 42;+ h = hash_mix(h + key.device);+ h = hash_mix(h + reinterpret_cast<size_t>(key.kernel));+ return h;+ }+ };++ static std::mutex cache_mutex;+ static std::unordered_map<cache_key, bool, cache_key_hash> cache;++ const cache_key key = { device, kernel };+ std::lock_guard<std::mutex> lock(cache_mutex);+ const auto it = cache.find(key);+ if (it != cache.end()) {+ return it->second;+ }++ cudaFuncAttributes attr = {};+ CUDA_CHECK(cudaFuncGetAttributes(&attr, kernel));++ // PDL device-side primitives are emitted only for PTX versions >= 90.+ // We have to guard on a loaded kernel's PTX version so a kernel forward-JIT'ed+ // from pre-Hopper PTX to a Hopper-or-newer GPU does not opt into PDL.+ const bool can_use_pdl = attr.ptxVersion >= 90;+ cache.emplace(key, can_use_pdl);+ return can_use_pdl;+}++#endif //defined(GGML_CUDA_USE_PDL)++// PDL and __restrict__ need to be mutually exclusive, see https://github.com/ggml-org/llama.cpp/pull/24030+# if (defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER)+# define GGML_CUDA_RESTRICT+# else+# define GGML_CUDA_RESTRICT __restrict__+# endif // defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER++template<typename Kernel, typename... Args>+static __inline__ void ggml_cuda_kernel_launch(Kernel kernel, const ggml_cuda_kernel_launch_params & launch_params, Args&&... args) {+#if defined(GGML_CUDA_USE_PDL)++ static const bool env_pdl_enabled = []() {+ const char * env = getenv("GGML_CUDA_PDL");+ return env == nullptr || std::atoi(env) != 0;+ }();++ if (env_pdl_enabled && ggml_cuda_kernel_can_use_pdl(reinterpret_cast<const void *>(kernel))) {+ auto pdl_cfg = ggml_cuda_pdl_config(launch_params);++ CUDA_CHECK(cudaLaunchKernelEx(&pdl_cfg.cfg, kernel, std::forward<Args>(args)... ));+ return;+ }+#endif //defined(GGML_CUDA_USE_PDL)++ kernel<<<launch_params.block_nums, launch_params.block_dims, launch_params.shmem, launch_params.stream>>>(std::forward<Args>(args)... );+ CUDA_CHECK(cudaGetLastError());+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cunew file mode 100644index 00000000..77fd2889--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cu@@ -0,0 +1,743 @@+#include "convert.cuh"+#include "dequantize.cuh"++#include <cstdint>++#define CUDA_Q8_0_NE_ALIGN 2048++template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static __global__ void dequantize_block(const void * __restrict__ vx, dst_t * __restrict__ y,+ const int64_t ne00, const int64_t ne01,+ const int64_t ne0203, const uint3 ne02,+ const int64_t s01, const int64_t s02, const int64_t s03) {+ const int64_t i00 = 2 * (int64_t(blockDim.x)*blockIdx.x + threadIdx.x);++ if (i00 >= ne00) {+ return;+ }++ for (int64_t i01 = blockIdx.y; i01 < ne01; i01 += gridDim.y) {+ for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {+ const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);+ const int64_t i02 = dm.y;+ const int64_t i03 = dm.x;++ const int64_t ibx0 = i03*s03 + i02*s02 + i01*s01;++ const int64_t ib = ibx0 + i00/qk; // block index+ const int64_t iqs = (i00%qk)/qr; // quant index+ const int64_t iybs = i00 - i00%qk; // y block start index+ const int64_t y_offset = qr == 1 ? 1 : qk/2;++ // dequantize+ float2 v;+ dequantize_kernel(vx, ib, iqs, v);++ const int64_t iy0 = (i0203*ne01 + i01)*ne00 + iybs + iqs;+ y[iy0 + 0] = ggml_cuda_cast<dst_t>(v.x);+ y[iy0 + y_offset] = ggml_cuda_cast<dst_t>(v.y);+ }+ }+}++template <bool need_check>+static __global__ void dequantize_block_q8_0_f16(const void * __restrict__ vx, half * __restrict__ y, const int64_t k) {+#if __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL+ constexpr int nint = CUDA_Q8_0_NE_ALIGN/sizeof(int) + WARP_SIZE;++ const int64_t i0 = CUDA_Q8_0_NE_ALIGN*blockIdx.x;+ const int * x0 = ((int *) vx) + blockIdx.x * nint;+ half2 * y2 = (half2 *) (y + i0);++ __shared__ int vals[nint];++#pragma unroll+ for (int ix0 = 0; ix0 < nint; ix0 += WARP_SIZE) {+ if (need_check && i0*sizeof(block_q8_0)/QK8_0 + sizeof(int)*(ix0 + threadIdx.x) >= k*sizeof(block_q8_0)/QK8_0) {+ break;+ }++ const int ix = ix0 + threadIdx.x;+ vals[ix] = x0[ix];+ }++ __syncthreads();++#pragma unroll+ for (int iy = 0; iy < CUDA_Q8_0_NE_ALIGN; iy += 2*WARP_SIZE) {+ if (need_check && i0 + iy + 2*threadIdx.x >= k) {+ return;+ }++ const half * b0 = ((const half *) vals) + (sizeof(block_q8_0)/sizeof(half)) * ((iy + 2*threadIdx.x)/QK8_0);+ const half d = *b0;+ const char2 qs = ((const char2 *) (b0 + 1))[threadIdx.x % (QK8_0/2)];++ y2[iy/2 + threadIdx.x] = __hmul2(make_half2(qs.x, qs.y), __half2half2(d));+ }+#else+ GGML_UNUSED_VARS(vx, y, k);+ NO_DEVICE_CODE;+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_PASCAL+}++template<typename dst_t>+static __global__ void dequantize_block_q4_0(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {++ const int64_t i = blockIdx.x;++ // assume 32 threads+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;+ const int64_t ib = 8*i + ir;+ if (ib >= nb32) {+ return;+ }++ dst_t * y = yy + 256*i + 32*ir + 4*il;++ const block_q4_0 * x = (const block_q4_0 *)vx + ib;+ const float d = __half2float(x->d);+ const float dm = -8*d;++ const uint8_t * q = x->qs + 4*il;++ for (int l = 0; l < 4; ++l) {+ y[l+ 0] = ggml_cuda_cast<dst_t>(d * (q[l] & 0xF) + dm);+ y[l+16] = ggml_cuda_cast<dst_t>(d * (q[l] >> 4) + dm);+ }+}++template<typename dst_t>+static __global__ void dequantize_block_q4_1(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {++ const int64_t i = blockIdx.x;++ // assume 32 threads+ const int64_t tid = threadIdx.x;+ const int64_t il = tid/8;+ const int64_t ir = tid%8;+ const int64_t ib = 8*i + ir;+ if (ib >= nb32) {+ return;+ }++ dst_t * y = yy + 256*i + 32*ir + 4*il;++ const block_q4_1 * x = (const block_q4_1 *)vx + ib;+ const float2 d = __half22float2(x->dm);++ const uint8_t * q = x->qs + 4*il;++ for (int l = 0; l < 4; ++l) {+ y[l+ 0] = ggml_cuda_cast<dst_t>(d.x * (q[l] & 0xF) + d.y);+ y[l+16] = ggml_cuda_cast<dst_t>(d.x * (q[l] >> 4) + d.y);+ }+}++//================================== k-quants++template<typename dst_t>+static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_q2_K(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_q3_K(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_q4_K(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_q5_K(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_q6_K(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_iq2_xxs(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_iq2_xs(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_iq2_s(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_iq3_xxs(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_iq3_s(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_iq1_s(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_iq1_m(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy, int nb32) {+ const int64_t i = blockIdx.x;++ const int64_t ib = 8*i + threadIdx.x%8;+ if (ib >= nb32) {+ return;+ }++ dequantize_iq4_nl(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_iq4_xs(vx, i, yy + i*QK_K, threadIdx.x);+}++template<typename dst_t>+static __global__ void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restrict__ yy) {+ const int64_t i = blockIdx.x;++ dequantize_mxfp4(vx, i, yy + i*QK_K, threadIdx.x);+}++template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static void dequantize_block_cuda(const void * vx, dst_t * y,+ const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,+ const int64_t s01, const int64_t s02, const int64_t s03, cudaStream_t stream) {+ const int64_t ne0203 = ne02*ne03;+ const uint3 ne02_fdv = init_fastdiv_values(ne02);+ const dim3 num_blocks((ne00 + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2*CUDA_DEQUANTIZE_BLOCK_SIZE), (int)std::min(ne01, (int64_t)65535), (int)std::min(ne0203, (int64_t)65535));+ dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>+ (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);+}++template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static void dequantize_block_cont_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) {+ dequantize_block_cuda<qk, qr, dequantize_kernel, dst_t>(vx, y, k, 1, 1, 1, k/qk, k/qk, k/qk, stream);+}++static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * __restrict__ y, const int64_t k, cudaStream_t stream) {+ const int num_blocks = (k + CUDA_Q8_0_NE_ALIGN - 1) / CUDA_Q8_0_NE_ALIGN;+ if (k % CUDA_Q8_0_NE_ALIGN == 0) {+ const bool need_check = false;+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);+ } else {+ const bool need_check = true;+ dequantize_block_q8_0_f16<need_check><<<num_blocks, WARP_SIZE, 0, stream>>>(vx, y, k);+ }+}++template<typename dst_t>+static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_q4_0_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_q4_0<<<nb, 32, 0, stream>>>(vx, y, nb32);+}++template<typename dst_t>+static void dequantize_row_q4_1_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb32 = k / 32;+ const int nb = (k + 255) / 256;+ dequantize_block_q4_1<<<nb, 32, 0, stream>>>(vx, y, nb32);+}++template<typename dst_t>+static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq2_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq2_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq2_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq3_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq3_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb32 = k / QK4_NL;+ const int nb = (k + QK_K - 1) / QK_K;+ dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y, nb32);+}++template<typename dst_t>+static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = k / QK_K;+ dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = (k + QK_K - 1) / QK_K;+ dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);+}++template<typename dst_t>+static void dequantize_row_mxfp4_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ const int nb = (k + QK_K - 1) / QK_K;+ dequantize_block_mxfp4<<<nb, 32, 0, stream>>>(vx, y);+}++template <typename dst_t>+static __global__ void dequantize_block_nvfp4(+ const void * __restrict__ vx,+ dst_t * __restrict__ yy,+ const int64_t ne) {+ const int64_t i = blockIdx.x;+ const int tid = threadIdx.x;++ const int64_t base = i * QK_NVFP4;+ if (base >= ne) {+ return;+ }++ const block_nvfp4 * x = (const block_nvfp4 *) vx;+ const block_nvfp4 & xb = x[i];++ const int sub = tid / (QK_NVFP4_SUB / 2);+ const int j = tid % (QK_NVFP4_SUB / 2);++ const float d = ggml_cuda_ue4m3_to_fp32(xb.d[sub]);+ const uint8_t q = xb.qs[sub * (QK_NVFP4_SUB / 2) + j];++ const int64_t y0 = base + sub * QK_NVFP4_SUB + j;+ const int64_t y1 = y0 + QK_NVFP4_SUB / 2;++ yy[y0] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q & 0x0F]);+ yy[y1] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q >> 4]);+}++template <typename dst_t>+static void dequantize_row_nvfp4_cuda(+ const void * vx,+ dst_t * y,+ const int64_t k,+ cudaStream_t stream) {+ GGML_ASSERT(k % QK_NVFP4 == 0);+ const int nb = k / QK_NVFP4;+ dequantize_block_nvfp4<<<nb, 32, 0, stream>>>(vx, y, k);+}+template <typename src_t, typename dst_t>+static __global__ void convert_unary(+ const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t ne00, const int64_t ne01,+ const int64_t ne0203, const uint3 ne02,+ const int64_t s01, const int64_t s02, const int64_t s03) {+ const int64_t i00 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;++ if (i00 >= ne00) {+ return;+ }++ const src_t * x = (const src_t *) vx;++ for (int64_t i01 = blockIdx.y; i01 < ne01; i01 += gridDim.y) {+ for (int64_t i0203 = blockIdx.z; i0203 < ne0203; i0203 += gridDim.z) {+ const uint2 dm = fast_div_modulo((uint32_t)i0203, ne02);+ const int64_t i02 = dm.y;+ const int64_t i03 = dm.x;++ const int64_t ix = i03*s03 + i02*s02 + i01*s01 + i00;+ const int64_t iy = (i0203*ne01 + i01)*ne00 + i00;+ y[iy] = ggml_cuda_cast<dst_t>(x[ix]);+ }+ }+}++template <typename T> struct alignas(sizeof(T)*4) cvt_vec4 { T v[4]; };++// four elements per thread, so a warp moves 512B (RDNA) / 1k (CDNA) per load+template <typename src_t, typename dst_t>+static __global__ void convert_unary_cont_vec4(+ const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k4) {+ const int64_t i = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;++ if (i >= k4) {+ return;+ }++ const cvt_vec4<src_t> xv = ((const cvt_vec4<src_t> *) vx)[i];++ cvt_vec4<dst_t> yv;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ yv.v[j] = ggml_cuda_cast<dst_t>(xv.v[j]);+ }++ ((cvt_vec4<dst_t> *) y)[i] = yv;+}++template <typename src_t, typename dst_t>+static void convert_unary_cuda(const void * vx, dst_t * y,+ const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,+ const int64_t s01, const int64_t s02, const int64_t s03, cudaStream_t stream) {+ const int64_t ne0203 = ne02*ne03;+ const uint3 ne02_fdv = init_fastdiv_values(ne02);+ const dim3 num_blocks((ne00 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE, (int)std::min(ne01, (int64_t)65535), (int)std::min(ne0203, (int64_t)65535));+ convert_unary<src_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>+ (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03);+}++template <typename src_t, typename dst_t>+static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) {+ if (k % 4 == 0 &&+ (uintptr_t) vx % alignof(cvt_vec4<src_t>) == 0 &&+ (uintptr_t) y % alignof(cvt_vec4<dst_t>) == 0) {+ const int64_t k4 = k/4;+ const int64_t num_blocks = (k4 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / CUDA_DEQUANTIZE_BLOCK_SIZE;+ convert_unary_cont_vec4<src_t, dst_t><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k4);+ return;+ }++ convert_unary_cuda<src_t>(vx, y, k, 1, 1, 1, k, k, k, stream);+}++to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q1_0:+ return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;+ case GGML_TYPE_Q4_0:+ return dequantize_row_q4_0_cuda;+ case GGML_TYPE_Q4_1:+ return dequantize_row_q4_1_cuda;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cont_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cont_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ return dequantize_block_cont_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_Q2_K:+ return dequantize_row_q2_K_cuda;+ case GGML_TYPE_Q3_K:+ return dequantize_row_q3_K_cuda;+ case GGML_TYPE_Q4_K:+ return dequantize_row_q4_K_cuda;+ case GGML_TYPE_Q5_K:+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;+ case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;+ case GGML_TYPE_IQ2_S:+ return dequantize_row_iq2_s_cuda;+ case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;+ case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;+ case GGML_TYPE_IQ1_M:+ return dequantize_row_iq1_m_cuda;+ case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;+ case GGML_TYPE_IQ4_XS:+ return dequantize_row_iq4_xs_cuda;+ case GGML_TYPE_IQ3_S:+ return dequantize_row_iq3_s_cuda;+ case GGML_TYPE_MXFP4:+ return dequantize_row_mxfp4_cuda;+ case GGML_TYPE_NVFP4:+ return dequantize_row_nvfp4_cuda;+ case GGML_TYPE_F32:+ return convert_unary_cont_cuda<float>;+ case GGML_TYPE_F16:+ return convert_unary_cont_cuda<half>;+ default:+ return nullptr;+ }+}++to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q1_0:+ return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;+ case GGML_TYPE_Q4_0:+ return dequantize_row_q4_0_cuda;+ case GGML_TYPE_Q4_1:+ return dequantize_row_q4_1_cuda;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cont_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cont_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ if (fp16_available(ggml_cuda_info().devices[ggml_cuda_get_device()].cc)) {+ return dequantize_block_q8_0_f16_cuda;+ }+ return dequantize_block_cont_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_Q2_K:+ return dequantize_row_q2_K_cuda;+ case GGML_TYPE_Q3_K:+ return dequantize_row_q3_K_cuda;+ case GGML_TYPE_Q4_K:+ return dequantize_row_q4_K_cuda;+ case GGML_TYPE_Q5_K:+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;+ case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;+ case GGML_TYPE_IQ2_S:+ return dequantize_row_iq2_s_cuda;+ case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;+ case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;+ case GGML_TYPE_IQ1_M:+ return dequantize_row_iq1_m_cuda;+ case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;+ case GGML_TYPE_IQ4_XS:+ return dequantize_row_iq4_xs_cuda;+ case GGML_TYPE_IQ3_S:+ return dequantize_row_iq3_s_cuda;+ case GGML_TYPE_MXFP4:+ return dequantize_row_mxfp4_cuda;+ case GGML_TYPE_NVFP4:+ return dequantize_row_nvfp4_cuda;+ case GGML_TYPE_F32:+ return convert_unary_cont_cuda<float>;+ case GGML_TYPE_BF16:+ return convert_unary_cont_cuda<nv_bfloat16>;+ default:+ return nullptr;+ }+}++to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q1_0:+ return dequantize_block_cont_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cont_cuda<QK2_0, QR2_0, dequantize_q2_0>;+ case GGML_TYPE_Q4_0:+ return dequantize_row_q4_0_cuda;+ case GGML_TYPE_Q4_1:+ return dequantize_row_q4_1_cuda;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cont_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cont_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ return dequantize_block_cont_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_Q2_K:+ return dequantize_row_q2_K_cuda;+ case GGML_TYPE_Q3_K:+ return dequantize_row_q3_K_cuda;+ case GGML_TYPE_Q4_K:+ return dequantize_row_q4_K_cuda;+ case GGML_TYPE_Q5_K:+ return dequantize_row_q5_K_cuda;+ case GGML_TYPE_Q6_K:+ return dequantize_row_q6_K_cuda;+ case GGML_TYPE_IQ2_XXS:+ return dequantize_row_iq2_xxs_cuda;+ case GGML_TYPE_IQ2_XS:+ return dequantize_row_iq2_xs_cuda;+ case GGML_TYPE_IQ2_S:+ return dequantize_row_iq2_s_cuda;+ case GGML_TYPE_IQ3_XXS:+ return dequantize_row_iq3_xxs_cuda;+ case GGML_TYPE_IQ1_S:+ return dequantize_row_iq1_s_cuda;+ case GGML_TYPE_IQ1_M:+ return dequantize_row_iq1_m_cuda;+ case GGML_TYPE_IQ4_NL:+ return dequantize_row_iq4_nl_cuda;+ case GGML_TYPE_IQ4_XS:+ return dequantize_row_iq4_xs_cuda;+ case GGML_TYPE_IQ3_S:+ return dequantize_row_iq3_s_cuda;+ case GGML_TYPE_MXFP4:+ return dequantize_row_mxfp4_cuda;+ case GGML_TYPE_NVFP4:+ return dequantize_row_nvfp4_cuda;+ case GGML_TYPE_F16:+ return convert_unary_cont_cuda<half>;+ case GGML_TYPE_BF16:+ return convert_unary_cont_cuda<nv_bfloat16>;+ default:+ return nullptr;+ }+}++to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_F32:+ return convert_unary_cuda<float>;+ case GGML_TYPE_Q1_0:+ return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;+ case GGML_TYPE_Q4_0:+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;+ case GGML_TYPE_Q4_1:+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_BF16:+ return convert_unary_cuda<nv_bfloat16>;+ default:+ return nullptr;+ }+}++to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_F32:+ return convert_unary_cuda<float, nv_bfloat16>;+ case GGML_TYPE_Q1_0:+ return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;+ case GGML_TYPE_Q4_0:+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;+ case GGML_TYPE_Q4_1:+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_F16:+ return convert_unary_cuda<half, nv_bfloat16>;+ default:+ return nullptr;+ }+}++to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_F16:+ return convert_unary_cuda<half, float>;+ case GGML_TYPE_Q1_0:+ return dequantize_block_cuda<QK1_0, QR1_0, dequantize_q1_0>;+ case GGML_TYPE_Q2_0:+ return dequantize_block_cuda<QK2_0, QR2_0, dequantize_q2_0>;+ case GGML_TYPE_Q4_0:+ return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;+ case GGML_TYPE_Q4_1:+ return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;+ case GGML_TYPE_Q5_0:+ return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;+ case GGML_TYPE_Q5_1:+ return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;+ case GGML_TYPE_Q8_0:+ return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;+ case GGML_TYPE_BF16:+ return convert_unary_cuda<nv_bfloat16, float>;+ default:+ return nullptr;+ }+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cuhnew file mode 100644index 00000000..91255ccf--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/convert.cuh@@ -0,0 +1,66 @@+#pragma once+#include "common.cuh"++#define CUDA_DEQUANTIZE_BLOCK_SIZE 256++template<typename T>+using to_t_cuda_t = void (*)(const void * x, T * y, int64_t k, cudaStream_t stream);++typedef to_t_cuda_t<float> to_fp32_cuda_t;+typedef to_t_cuda_t<half> to_fp16_cuda_t;+typedef to_t_cuda_t<nv_bfloat16> to_bf16_cuda_t;++to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);++to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type);++to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);++// TODO more general support for non-contiguous inputs++template<typename T>+using to_t_nc_cuda_t = void (*)(const void * x, T * y,+ int64_t ne00, int64_t ne01, int64_t ne02, int64_t ne03,+ int64_t s01, int64_t s02, int64_t s03, cudaStream_t stream);++typedef to_t_nc_cuda_t<float> to_fp32_nc_cuda_t;+typedef to_t_nc_cuda_t<half> to_fp16_nc_cuda_t;+typedef to_t_nc_cuda_t<nv_bfloat16> to_bf16_nc_cuda_t;++to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type);+to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type);+to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type);++template<typename dst_t, typename src_t>+ __host__ __device__ inline dst_t ggml_cuda_cast(src_t x) {+ if constexpr (std::is_same_v<dst_t, src_t>) {+ return x;+ } else if constexpr(std::is_same_v<dst_t, nv_bfloat16>) {+ return __float2bfloat16(float(x));+ } else if constexpr(std::is_same_v<src_t, nv_bfloat16>) {+ return __bfloat162float(x);+ } else if constexpr(std::is_same_v<src_t, float2> && std::is_same_v<dst_t, half2>) {+ return __float22half2_rn(x);+ } else if constexpr(std::is_same_v<src_t, nv_bfloat162> && std::is_same_v<dst_t, float2>) {+#ifdef GGML_USE_HIP+ return make_float2(__bfloat162float(__low2bfloat16(x)), __bfloat162float(__high2bfloat16(x)));+#else+#if !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= 800+ return __bfloat1622float2(x);+#else+ return make_float2(__bfloat162float(x.x), __bfloat162float(x.y));+#endif // !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= 800+#endif // GGML_USE_HIP+ } else if constexpr(std::is_same_v<src_t, float2> && std::is_same_v<dst_t, nv_bfloat162>) {+ // bypass compile error on cuda 12.0.1+#ifdef GGML_USE_HIP+ return __float22bfloat162_rn(x);+#else+ return {x.x, x.y};+#endif // GGML_USE_HIP+ } else if constexpr(std::is_same_v<dst_t, int32_t>) {+ return int32_t(x);+ } else {+ return float(x);+ }+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/dequantize.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/dequantize.cuhnew file mode 100644index 00000000..651524be--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/dequantize.cuh@@ -0,0 +1,452 @@+#include "common.cuh"+#include "convert.cuh"++static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const int64_t ib, const int iqs, float2 & v){+ const block_q1_0 * x = (const block_q1_0 *) vx;++ const float d = x[ib].d;++ const int bit_index_0 = iqs;+ const int bit_index_1 = iqs + 1;++ const int byte_index_0 = bit_index_0 / 8;+ const int bit_offset_0 = bit_index_0 % 8;++ const int byte_index_1 = bit_index_1 / 8;+ const int bit_offset_1 = bit_index_1 % 8;++ // Extract bits: 1 = +d, 0 = -d (branchless)+ const int bit_0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 1;+ const int bit_1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 1;++ v.x = (2*bit_0 - 1) * d;+ v.y = (2*bit_1 - 1) * d;+}++static __device__ __forceinline__ void dequantize_q2_0(const void * vx, const int64_t ib, const int iqs, float2 & v){+ const block_q2_0 * x = (const block_q2_0 *) vx;++ const float d = x[ib].d;++ // Q2_0: 2 bits per element, 4 elements per byte.+ // Stored code c in {0,1,2,3} maps to symbol s = c - 1 in {-1, 0, +1, +2}.+ const int byte_index_0 = iqs / 4;+ const int bit_offset_0 = (iqs % 4) * 2;++ const int byte_index_1 = (iqs + 1) / 4;+ const int bit_offset_1 = ((iqs + 1) % 4) * 2;++ const int c0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 0x3;+ const int c1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 0x3;++ v.x = (c0 - 1) * d;+ v.y = (c1 - 1) * d;+}++static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){+ const block_q4_0 * x = (const block_q4_0 *) vx;++ const float d = x[ib].d;++ const int vui = x[ib].qs[iqs];++ v.x = vui & 0xF;+ v.y = vui >> 4;++ v.x = (v.x - 8.0f) * d;+ v.y = (v.y - 8.0f) * d;+}++static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, float2 & v){+ const block_q4_1 * x = (const block_q4_1 *) vx;++ const float2 dm = __half22float2(x[ib].dm);++ const int vui = x[ib].qs[iqs];++ v.x = vui & 0xF;+ v.y = vui >> 4;++ v.x = (v.x * dm.x) + dm.y;+ v.y = (v.y * dm.x) + dm.y;+}++static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, float2 & v){+ const block_q5_0 * x = (const block_q5_0 *) vx;++ const float d = x[ib].d;++ uint32_t qh;+ memcpy(&qh, x[ib].qh, sizeof(qh));++ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;++ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);++ v.x = (v.x - 16.0f) * d;+ v.y = (v.y - 16.0f) * d;+}++static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, float2 & v){+ const block_q5_1 * x = (const block_q5_1 *) vx;++ const float2 dm = __half22float2(x[ib].dm);++ uint32_t qh;+ memcpy(&qh, x[ib].qh, sizeof(qh));++ const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;+ const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;++ v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);+ v.y = ((x[ib].qs[iqs] >> 4) | xh_1);++ v.x = (v.x * dm.x) + dm.y;+ v.y = (v.y * dm.x) + dm.y;+}++static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, float2 & v){+ const block_q8_0 * x = (const block_q8_0 *) vx;++ const float d = x[ib].d;++ v.x = x[ib].qs[iqs + 0];+ v.y = x[ib].qs[iqs + 1];++ v.x *= d;+ v.y *= d;+}++//================================== k-quants++// Each call dequantizes one super-block of QK_K values into y using the+// thread layout of the caller: 32 threads for q4_K, 64 threads otherwise.++template<typename dst_t>+static __device__ __forceinline__ void dequantize_q2_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {+ const block_q2_K * x = (const block_q2_K *) vx;++ const int64_t n = tid/32;+ const int64_t l = tid - 32*n;+ const int64_t is = 8*n + l/16;++ const uint8_t q = x[ib].qs[32*n + l];+ dst_t * y = yy + 128*n;++ float dall = __low2half(x[ib].dm);+ float dmin = __high2half(x[ib].dm);+ y[l+ 0] = ggml_cuda_cast<dst_t>(dall * (x[ib].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[ib].scales[is+0] >> 4));+ y[l+32] = ggml_cuda_cast<dst_t>(dall * (x[ib].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[ib].scales[is+2] >> 4));+ y[l+64] = ggml_cuda_cast<dst_t>(dall * (x[ib].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[ib].scales[is+4] >> 4));+ y[l+96] = ggml_cuda_cast<dst_t>(dall * (x[ib].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[ib].scales[is+6] >> 4));+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_q3_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {+ const block_q3_K * x = (const block_q3_K *) vx;++ const int64_t r = tid/4;+ const int64_t t = r/2;+ const int64_t is0 = r%2;+ const int64_t l0 = 16*is0 + 4*(tid%4);+ const int64_t n = t / 4;+ const int64_t j = t - 4*n;++ uint8_t m = 1 << (4*n + j);+ int64_t is = 8*n + 2*j + is0;+ int shift = 2*j;++ int8_t us = is < 4 ? (x[ib].scales[is-0] & 0xF) | (((x[ib].scales[is+8] >> 0) & 3) << 4) :+ is < 8 ? (x[ib].scales[is-0] & 0xF) | (((x[ib].scales[is+4] >> 2) & 3) << 4) :+ is < 12 ? (x[ib].scales[is-8] >> 4) | (((x[ib].scales[is+0] >> 4) & 3) << 4) :+ (x[ib].scales[is-8] >> 4) | (((x[ib].scales[is-4] >> 6) & 3) << 4);+ float d_all = x[ib].d;+ float dl = d_all * (us - 32);++ dst_t * y = yy + 128*n + 32*j;+ const uint8_t * q = x[ib].qs + 32*n;+ const uint8_t * hm = x[ib].hmask;++ for (int l = l0; l < l0+4; ++l) {+ y[l] = ggml_cuda_cast<dst_t>(dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)));+ }+}++static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) {+ if (j < 4) {+ d = q[j] & 63; m = q[j + 4] & 63;+ } else {+ d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);+ m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_q4_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {+ const block_q4_K * x = (const block_q4_K *) vx;++ // assume 32 threads+ const int64_t il = tid/8;+ const int64_t ir = tid%8;+ const int64_t is = 2*il;+ const int64_t n = 4;++ dst_t * y = yy + 64*il + n*ir;++ const float dall = __low2half(x[ib].dm);+ const float dmin = __high2half(x[ib].dm);++ const uint8_t * q = x[ib].qs + 32*il + n*ir;++ uint8_t sc, m;+ get_scale_min_k4(is + 0, x[ib].scales, sc, m);+ const float d1 = dall * sc; const float m1 = dmin * m;+ get_scale_min_k4(is + 1, x[ib].scales, sc, m);+ const float d2 = dall * sc; const float m2 = dmin * m;+ for (int l = 0; l < n; ++l) {+ y[l + 0] = ggml_cuda_cast<dst_t>(d1 * (q[l] & 0xF) - m1);+ y[l +32] = ggml_cuda_cast<dst_t>(d2 * (q[l] >> 4) - m2);+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_q5_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {+ const block_q5_K * x = (const block_q5_K *) vx;++ // assume 64 threads - this is very slightly better than the one below+ const int64_t il = tid/16; // il is in 0...3+ const int64_t ir = tid%16; // ir is in 0...15+ const int64_t is = 2*il; // is is in 0...6++ dst_t * y = yy + 64*il + 2*ir;++ const float dall = __low2half(x[ib].dm);+ const float dmin = __high2half(x[ib].dm);++ const uint8_t * ql = x[ib].qs + 32*il + 2*ir;+ const uint8_t * qh = x[ib].qh + 2*ir;++ uint8_t sc, m;+ get_scale_min_k4(is + 0, x[ib].scales, sc, m);+ const float d1 = dall * sc; const float m1 = dmin * m;+ get_scale_min_k4(is + 1, x[ib].scales, sc, m);+ const float d2 = dall * sc; const float m2 = dmin * m;++ uint8_t hm = 1 << (2*il);+ y[ 0] = ggml_cuda_cast<dst_t>(d1 * ((ql[ 0] & 0xF) + (qh[ 0] & hm ? 16 : 0)) - m1);+ y[ 1] = ggml_cuda_cast<dst_t>(d1 * ((ql[ 1] & 0xF) + (qh[ 1] & hm ? 16 : 0)) - m1);+ hm <<= 1;+ y[32] = ggml_cuda_cast<dst_t>(d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2);+ y[33] = ggml_cuda_cast<dst_t>(d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2);+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_q6_K(const void * vx, const int64_t ib, dst_t * yy, const int tid) {+ const block_q6_K * x = (const block_q6_K *) vx;++ // assume 64 threads - this is very slightly better than the one below+ const int64_t ip = tid/32; // ip is 0 or 1+ const int64_t il = tid - 32*ip; // 0...32+ const int64_t is = 8*ip + il/16;++ dst_t * y = yy + 128*ip + il;++ const float d = x[ib].d;++ const uint8_t * ql = x[ib].ql + 64*ip + il;+ const uint8_t qh = x[ib].qh[32*ip + il];+ const int8_t * sc = x[ib].scales + is;++ y[ 0] = ggml_cuda_cast<dst_t>(d * sc[0] * ((int8_t)((ql[ 0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32));+ y[32] = ggml_cuda_cast<dst_t>(d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32));+ y[64] = ggml_cuda_cast<dst_t>(d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32));+ y[96] = ggml_cuda_cast<dst_t>(d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32));+}++//================================== i-quants++// Each call dequantizes one super-block of QK_K values into y with 32+// threads; iq4_nl packs QK_K/QK4_NL sub-blocks per super-block.++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq2_xxs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_iq2_xxs * x = (const block_iq2_xxs *) vx;++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 8*il;+ const uint16_t * q2 = x[ibs].qs + 4*ib;+ const uint8_t * aux8 = (const uint8_t *)q2;+ const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[il]);+ const uint32_t aux32 = q2[2] | (q2[3] << 16);+ const float d = (float)x[ibs].d * (0.5f + (aux32 >> 28)) * 0.25f;+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];+ for (int j = 0; j < 8; ++j) {+ y[j] = ggml_cuda_cast<dst_t>(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq2_xs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_iq2_xs * x = (const block_iq2_xs *) vx;++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 8*il;+ const uint16_t * q2 = x[ibs].qs + 4*ib;+ const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (q2[il] & 511));+ const float d = (float)x[ibs].d * (0.5f + ((x[ibs].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;+ const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];+ for (int j = 0; j < 8; ++j) {+ y[j] = ggml_cuda_cast<dst_t>(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq2_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_iq2_s * x = (const block_iq2_s *) vx;++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 8*il;+ const uint8_t * grid = (const uint8_t *)(iq2s_grid + (x[ibs].qs[4*ib+il] | ((x[ibs].qh[ib] << (8-2*il)) & 0x300)));+ const float d = (float)x[ibs].d * (0.5f + ((x[ibs].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f;+ const uint8_t signs = x[ibs].qs[QK_K/8+4*ib+il];+ for (int j = 0; j < 8; ++j) {+ y[j] = ggml_cuda_cast<dst_t>(d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f));+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq3_xxs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_iq3_xxs * x = (const block_iq3_xxs *) vx;++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 8*il;+ const uint8_t * q3 = x[ibs].qs + 8*ib;+ const uint16_t * gas = (const uint16_t *)(x[ibs].qs + QK_K/4) + 2*ib;+ const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + q3[2*il+0]);+ const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + q3[2*il+1]);+ const uint32_t aux32 = gas[0] | (gas[1] << 16);+ const float d = (float)x[ibs].d * (0.5f + (aux32 >> 28)) * 0.5f;+ const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127];+ for (int j = 0; j < 4; ++j) {+ y[j+0] = ggml_cuda_cast<dst_t>(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f));+ y[j+4] = ggml_cuda_cast<dst_t>(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f));+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq3_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_iq3_s * x = (const block_iq3_s *) vx;++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 8*il;+ const uint8_t * qs = x[ibs].qs + 8*ib;+ const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*il+0] | ((x[ibs].qh[ib] << (8-2*il)) & 256)));+ const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*il+1] | ((x[ibs].qh[ib] << (7-2*il)) & 256)));+ const float d = (float)x[ibs].d * (1 + 2*((x[ibs].scales[ib/2] >> 4*(ib%2)) & 0xf));+ const uint8_t signs = x[ibs].signs[4*ib + il];+ for (int j = 0; j < 4; ++j) {+ y[j+0] = ggml_cuda_cast<dst_t>(d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f));+ y[j+4] = ggml_cuda_cast<dst_t>(d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f));+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq1_s(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_iq1_s * x = (const block_iq1_s *) vx;++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 8*il;+ const float delta = x[ibs].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;+ const float d = (float)x[ibs].d * (2*((x[ibs].qh[ib] >> 12) & 7) + 1);+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;+ grid32[0] = iq1s_grid_gpu[x[ibs].qs[4*ib+il] | (((x[ibs].qh[ib] >> 3*il) & 7) << 8)];+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;+ grid32[0] &= 0x0f0f0f0f;+ for (int j = 0; j < 8; ++j) {+ y[j] = ggml_cuda_cast<dst_t>(d * (q[j] + delta));+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq1_m(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_iq1_m * x = (const block_iq1_m *) vx;++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 8*il;+ const uint16_t * sc = (const uint16_t *)x[ibs].scales;+ iq1m_scale_t scale;+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);+ const int64_t ib16 = 2*ib + il/2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);+ const float d = (float)scale.f16 * (2*((sc[ib16/4] >> 3*(ib16%4)) & 0x7) + 1);+ const float delta = x[ibs].qh[2*ib+il/2] & (0x08 << 4*(il%2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;+ uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32;+ grid32[0] = iq1s_grid_gpu[x[ibs].qs[4*ib+il] | (((x[ibs].qh[2*ib+il/2] >> 4*(il%2)) & 7) << 8)];+ grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;+ grid32[0] &= 0x0f0f0f0f;+ for (int j = 0; j < 8; ++j) {+ y[j] = ggml_cuda_cast<dst_t>(d * (q[j] + delta));+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq4_nl(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_iq4_nl * x = (const block_iq4_nl *) vx + ibs*(QK_K/QK4_NL);++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 4*il;+ const uint8_t * q4 = x[ib].qs + 4*il;+ const float d = (float)x[ib].d;+ for (int j = 0; j < 4; ++j) {+ y[j+ 0] = ggml_cuda_cast<dst_t>(d * kvalues_iq4nl[q4[j] & 0xf]);+ y[j+16] = ggml_cuda_cast<dst_t>(d * kvalues_iq4nl[q4[j] >> 4]);+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_iq4_xs(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {+ const block_iq4_xs * x = (const block_iq4_xs *)vx;++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 4*il;+ const uint8_t * q4 = x[ibs].qs + 16*ib + 4*il;+ const float d = (float)x[ibs].d * ((((x[ibs].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[ibs].scales_h >> 2*ib) & 3) << 4)) - 32);+ for (int j = 0; j < 4; ++j) {+ y[j+ 0] = ggml_cuda_cast<dst_t>(d * kvalues_iq4nl[q4[j] & 0xf]);+ y[j+16] = ggml_cuda_cast<dst_t>(d * kvalues_iq4nl[q4[j] >> 4]);+ }+}++template<typename dst_t>+static __device__ __forceinline__ void dequantize_mxfp4(const void * vx, const int64_t ibs, dst_t * yy, const int tid) {++ const block_mxfp4 * x = (const block_mxfp4 *) vx + ibs*(QK_K/QK_MXFP4);++ const int64_t il = tid/8; // 0...3+ const int64_t ib = tid%8; // 0...7+ dst_t * y = yy + 32*ib + 4*il;+ const uint8_t * q4 = x[ib].qs + 4*il;+ const float d = ggml_cuda_e8m0_to_fp32(x[ib].e);+ for (int j = 0; j < 4; ++j) {+ y[j+ 0] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q4[j] & 0xf]*0.5f);+ y[j+16] = ggml_cuda_cast<dst_t>(d * kvalues_mxfp4[q4[j] >> 4]*0.5f);+ }+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cunew file mode 100644index 00000000..d97e8fdb--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cu@@ -0,0 +1,488 @@+#include "getrows.cuh"+#include "dequantize.cuh"+#include "convert.cuh"++template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>+static __global__ void k_get_rows(+ const void * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst,+ const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/+ /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/+ /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3,+ /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03,+ const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) {++ ggml_cuda_pdl_sync();+ for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) {+ for (int64_t i00 = 2*(blockIdx.y*blockDim.x + threadIdx.x); i00 < ne00; i00 += gridDim.y*blockDim.x) {+ // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher.+ const int i10 = blockIdx.x;+ const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv);+ const int i11 = dm.x;+ const int i12 = dm.y;++ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];++ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const void * src0_row = (const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03;++ const int ib = i00/qk; // block index+ const int iqs = (i00%qk)/qr; // quant index+ const int iybs = i00 - i00%qk; // dst block start index+ const int y_offset = qr == 1 ? 1 : qk/2;++ // dequantize+ float2 v;+ dequantize_kernel(src0_row, ib, iqs, v);++ dst_row[iybs + iqs + 0] = ggml_cuda_cast<dst_t>(v.x);+ dst_row[iybs + iqs + y_offset] = ggml_cuda_cast<dst_t>(v.y);+ }+ }+}++template<typename dst_t, dequantize_kq_t<dst_t> dequantize_kq>+static __global__ void k_get_rows_kq(+ const void * __restrict__ src0, const int32_t * __restrict__ src1, dst_t * __restrict__ dst,+ const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/+ /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/+ /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3,+ /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03,+ const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) {++ ggml_cuda_pdl_sync();+ const int64_t nsb = ne00/QK_K; // super-blocks per row+ for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) {+ // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher.+ const int i10 = blockIdx.x;+ const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv);+ const int i11 = dm.x;+ const int i12 = dm.y;++ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];++ dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const void * src0_row = (const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03;++ for (int64_t ib = blockIdx.y; ib < nsb; ib += gridDim.y) {+ dequantize_kq(src0_row, ib, dst_row + ib*QK_K, threadIdx.x);+ }+ }+}++template<typename src0_t, typename dst_t>+static __global__ void k_get_rows_float(+ const src0_t * src0_ptr, const int32_t * src1_ptr, dst_t * dst_ptr,+ const int64_t ne00, /*const int64_t ne01, const int64_t ne02, const int64_t ne03,*/+ /*const int64_t ne10,*/ const int64_t ne11, const uint3 ne12_fdv, /*const int64_t ne13,*/+ /*const size_t s0,*/ const size_t s1, const size_t s2, const size_t s3,+ /*const size_t nb00,*/ const size_t nb01, const size_t nb02, const size_t nb03,+ const size_t s10, const size_t s11, const size_t s12/*, const size_t s13*/) {++ ggml_cuda_pdl_lc();+ const src0_t * GGML_CUDA_RESTRICT src0 = src0_ptr;+ const int32_t * GGML_CUDA_RESTRICT src1 = src1_ptr;+ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;+ ggml_cuda_pdl_sync();+ for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) {+ // The x and y dimensions of the grid are swapped because the maximum allowed grid size for x is higher.+ const int i10 = blockIdx.x;+ const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv);+ const int i11 = dm.x;+ const int i12 = dm.y;++ const int i01 = src1[i10*s10 + i11*s11 + i12*s12];++ dst_t * GGML_CUDA_RESTRICT dst_row = dst + i10*s1 + i11*s2 + i12*s3;+ const src0_t * GGML_CUDA_RESTRICT src0_row = (const src0_t *)((const char *) src0 + i01*nb01 + i11*nb02 + i12*nb03);++ for (int64_t i00 = blockIdx.y*blockDim.x + threadIdx.x; i00 < ne00; i00 += gridDim.y*blockDim.x) {+ dst_row[i00] = ggml_cuda_cast<dst_t>(src0_row[i00]);+ }+ }+}++template<typename dst_t>+static __global__ void k_get_rows_float_vec(+ const dst_t * src0_ptr, const int32_t * src1_ptr, dst_t * dst_ptr,+ const int64_t ne00v,+ const int64_t ne11, const uint3 ne12_fdv,+ const size_t s1, const size_t s2, const size_t s3,+ const size_t nb01, const size_t nb02, const size_t nb03,+ const size_t s10, const size_t s11, const size_t s12) {++ ggml_cuda_pdl_lc();+ ggml_cuda_pdl_sync();+ for (int64_t z = blockIdx.z; z < ne11*(int64_t)ne12_fdv.z; z += gridDim.z) {+ const int i10 = blockIdx.x;+ const uint2 dm = fast_div_modulo((uint32_t)z, ne12_fdv);+ const int i11 = dm.x;+ const int i12 = dm.y;++ const int i01 = src1_ptr[i10*s10 + i11*s11 + i12*s12];++ int4 * GGML_CUDA_RESTRICT dst_row = (int4 *) (dst_ptr + i10*s1 + i11*s2 + i12*s3);+ const int4 * GGML_CUDA_RESTRICT src0_row = (const int4 *)((const char *) src0_ptr + i01*nb01 + i11*nb02 + i12*nb03);++ for (int64_t i = blockIdx.y*blockDim.x + threadIdx.x; i < ne00v; i += gridDim.y*blockDim.x) {+ dst_row[i] = src0_row[i];+ }+ }+}++template<typename grad_t, typename dst_t>+static __global__ void k_get_rows_back_float(+ const grad_t * __restrict__ grad, const int32_t * __restrict__ rows, dst_t * __restrict__ dst,+ const int64_t ncols, const int64_t nrows_grad, const int64_t nrows_dst) {+ const int col = blockIdx.x*blockDim.x + threadIdx.x;++ if (col >= ncols) {+ return;+ }++ ggml_cuda_pdl_sync();++ // grid.y is clamped to the CUDA grid limit, so stride over the destination rows+ for (int64_t dst_row = blockIdx.y; dst_row < nrows_dst; dst_row += gridDim.y) {+ float sum = 0.0f;++ for (int64_t i = 0; i < nrows_grad; ++i) {+ if (rows[i] != dst_row) {+ continue;+ }+ sum += grad[i*ncols + col];+ }++ dst[dst_row*ncols + col] = sum;+ }+}++template<int qk, int qr, dequantize_kernel_t dq, typename dst_t>+static void get_rows_cuda_q(+ const void * src0_d, const int32_t * src1_d, dst_t * dst_d,+ const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,+ const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,+ const size_t nb1, const size_t nb2, const size_t nb3,+ cudaStream_t stream) {+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);+ const int block_num_y = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);+ const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));++ // strides in elements+ // const size_t s0 = nb0 / sizeof(dst_t);+ const size_t s1 = nb1 / sizeof(dst_t);+ const size_t s2 = nb2 / sizeof(dst_t);+ const size_t s3 = nb3 / sizeof(dst_t);++ const size_t s10 = nb10 / sizeof(int32_t);+ const size_t s11 = nb11 / sizeof(int32_t);+ const size_t s12 = nb12 / sizeof(int32_t);+ // const size_t s13 = nb13 / sizeof(int32_t);++ GGML_ASSERT(ne00 % 2 == 0);++ GGML_ASSERT(ne12 > 0);+ GGML_ASSERT(ne11 <= std::numeric_limits<uint32_t>::max() / ne12);+ const uint3 ne12_fdv = init_fastdiv_values(ne12);++ k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(+ src0_d, src1_d, dst_d,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10,*/ ne11, ne12_fdv, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);+}++template<int block_dim, typename dst_t, dequantize_kq_t<dst_t> dequantize_kq>+static void get_rows_cuda_kq(+ const void * src0_d, const int32_t * src1_d, dst_t * dst_d,+ const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,+ const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,+ const size_t nb1, const size_t nb2, const size_t nb3,+ cudaStream_t stream) {+ GGML_ASSERT(ne00 % QK_K == 0);+ const int64_t nsb = ne00/QK_K;++ const dim3 block_dims(block_dim, 1, 1);+ const dim3 block_nums(ne10, MIN(nsb, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));++ // strides in elements+ // const size_t s0 = nb0 / sizeof(dst_t);+ const size_t s1 = nb1 / sizeof(dst_t);+ const size_t s2 = nb2 / sizeof(dst_t);+ const size_t s3 = nb3 / sizeof(dst_t);++ const size_t s10 = nb10 / sizeof(int32_t);+ const size_t s11 = nb11 / sizeof(int32_t);+ const size_t s12 = nb12 / sizeof(int32_t);+ // const size_t s13 = nb13 / sizeof(int32_t);++ GGML_ASSERT(ne12 > 0);+ GGML_ASSERT(ne11 <= std::numeric_limits<uint32_t>::max() / ne12);+ const uint3 ne12_fdv = init_fastdiv_values(ne12);++ k_get_rows_kq<dst_t, dequantize_kq><<<block_nums, block_dims, 0, stream>>>(+ src0_d, src1_d, dst_d,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10,*/ ne11, ne12_fdv, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);+}++template<typename src0_t, typename dst_t>+static void get_rows_cuda_float(+ const src0_t * src0_d, const int32_t * src1_d, dst_t * dst_d,+ const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,+ const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,+ const size_t nb1, const size_t nb2, const size_t nb3,+ cudaStream_t stream) {+ const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);++ // strides in elements+ // const size_t s0 = nb0 / sizeof(dst_t);+ const size_t s1 = nb1 / sizeof(dst_t);+ const size_t s2 = nb2 / sizeof(dst_t);+ const size_t s3 = nb3 / sizeof(dst_t);++ const size_t s10 = nb10 / sizeof(int32_t);+ const size_t s11 = nb11 / sizeof(int32_t);+ const size_t s12 = nb12 / sizeof(int32_t);+ // const size_t s13 = nb13 / sizeof(int32_t);++ GGML_ASSERT(ne12 > 0);+ GGML_ASSERT(ne11 <= std::numeric_limits<uint32_t>::max() / ne12);+ const uint3 ne12_fdv = init_fastdiv_values(ne12);++ if constexpr (std::is_same<src0_t, dst_t>::value) {+ constexpr int VEC = 16 / sizeof(dst_t);+ const int64_t ne00v = ne00 / VEC;+ const int64_t vec_block_num_y = (ne00v + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;+ const bool enough_blocks = vec_block_num_y * ne10 * ne11 * ne12 >= 128;+ const bool can_vec = VEC > 1 && enough_blocks &&+ (ne00 % VEC == 0) &&+ (nb01 % 16 == 0) && (nb02 % 16 == 0) && (nb03 % 16 == 0) &&+ (nb1 % 16 == 0) && (nb2 % 16 == 0) && (nb3 % 16 == 0) &&+ (((uintptr_t) src0_d) % 16 == 0) && (((uintptr_t) dst_d) % 16 == 0);++ if (can_vec) {+ const int block_num_y = vec_block_num_y;+ const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{block_nums, block_dims, 0, stream};+ ggml_cuda_kernel_launch(k_get_rows_float_vec<dst_t>, launch_params,+ (const dst_t *) src0_d, src1_d, dst_d,+ ne00v, ne11, ne12_fdv,+ s1, s2, s3,+ nb01, nb02, nb03,+ s10, s11, s12);+ return;+ }+ }++ const int block_num_y = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;+ const dim3 block_nums(ne10, MIN(block_num_y, UINT16_MAX), MIN(ne11*ne12, UINT16_MAX));++ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{block_nums, block_dims, 0, stream};+ ggml_cuda_kernel_launch(k_get_rows_float<src0_t, dst_t>, launch_params,+ src0_d, src1_d, dst_d,+ ne00, /*ne01, ne02, ne03,*/+ /*ne10,*/ ne11, ne12_fdv, /*ne13,*/+ /* s0,*/ s1, s2, s3,+ /* nb00,*/ nb01, nb02, nb03,+ s10, s11, s12/*, s13*/);+}++template <typename dst_t>+static void ggml_cuda_get_rows_switch_src0_type(+ const void * src0_d, const ggml_type src0_type, const int32_t * src1_d, dst_t * dst_d,+ const int64_t ne00, const size_t nb01, const size_t nb02, const size_t nb03,+ const int64_t ne10, const int64_t ne11, const int64_t ne12, const size_t nb10, const size_t nb11, const size_t nb12,+ const size_t nb1, const size_t nb2, const size_t nb3,+ cudaStream_t stream) {+ switch (src0_type) {+ case GGML_TYPE_F16:+ get_rows_cuda_float((const half *) src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_F32:+ get_rows_cuda_float((const float *) src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_I32:+ get_rows_cuda_float((const int32_t *) src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_BF16:+ get_rows_cuda_float((const nv_bfloat16 *) src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q1_0:+ get_rows_cuda_q<QK1_0, QR1_0, dequantize_q1_0>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q2_0:+ get_rows_cuda_q<QK2_0, QR2_0, dequantize_q2_0>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q4_0:+ get_rows_cuda_q<QK4_0, QR4_0, dequantize_q4_0>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q4_1:+ get_rows_cuda_q<QK4_1, QR4_1, dequantize_q4_1>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q5_0:+ get_rows_cuda_q<QK5_0, QR5_0, dequantize_q5_0>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q5_1:+ get_rows_cuda_q<QK5_1, QR5_1, dequantize_q5_1>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q8_0:+ get_rows_cuda_q<QK8_0, QR8_0, dequantize_q8_0>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q2_K:+ get_rows_cuda_kq<64, dst_t, dequantize_q2_K<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q3_K:+ get_rows_cuda_kq<64, dst_t, dequantize_q3_K<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q4_K:+ get_rows_cuda_kq<32, dst_t, dequantize_q4_K<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q5_K:+ get_rows_cuda_kq<64, dst_t, dequantize_q5_K<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_Q6_K:+ get_rows_cuda_kq<64, dst_t, dequantize_q6_K<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ2_XXS:+ get_rows_cuda_kq<32, dst_t, dequantize_iq2_xxs<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ2_XS:+ get_rows_cuda_kq<32, dst_t, dequantize_iq2_xs<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ2_S:+ get_rows_cuda_kq<32, dst_t, dequantize_iq2_s<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ3_XXS:+ get_rows_cuda_kq<32, dst_t, dequantize_iq3_xxs<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ3_S:+ get_rows_cuda_kq<32, dst_t, dequantize_iq3_s<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ1_S:+ get_rows_cuda_kq<32, dst_t, dequantize_iq1_s<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ1_M:+ get_rows_cuda_kq<32, dst_t, dequantize_iq1_m<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ4_NL:+ get_rows_cuda_kq<32, dst_t, dequantize_iq4_nl<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_IQ4_XS:+ get_rows_cuda_kq<32, dst_t, dequantize_iq4_xs<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_MXFP4:+ get_rows_cuda_kq<32, dst_t, dequantize_mxfp4<dst_t>>(src0_d, src1_d, dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ default:+ GGML_ABORT("%s: unsupported src0 type: %s\n", __func__, ggml_type_name(src0_type));+ }+}++void get_rows_cuda(+ const void * src0_d, ggml_type src0_type, const int32_t * src1_d, void * dst_d, ggml_type dst_type,+ int64_t ne00, size_t nb01, size_t nb02, size_t nb03,+ int64_t ne10, int64_t ne11, int64_t ne12, size_t nb10, size_t nb11, size_t nb12,+ size_t nb1, size_t nb2, size_t nb3,+ cudaStream_t stream) {+ switch (dst_type) {+ case GGML_TYPE_F32:+ ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (float *) dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_I32:+ ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (int32_t *) dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_F16:+ ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (half *) dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ case GGML_TYPE_BF16:+ ggml_cuda_get_rows_switch_src0_type(src0_d, src0_type, src1_d, (nv_bfloat16 *) dst_d,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+ break;+ default:+ GGML_ABORT("%s: unsupported dst type: %s\n", __func__, ggml_type_name(dst_type));+ }+}++void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0];+ const ggml_tensor * src1 = dst->src[1];++ cudaStream_t stream = ctx.stream();++ GGML_TENSOR_BINARY_OP_LOCALS++ GGML_ASSERT(src1->type == GGML_TYPE_I32);+ GGML_ASSERT(ne13 == 1);++ GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));+ GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));+ GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));++ get_rows_cuda(src0->data, src0->type, (const int32_t *) src1->data, dst->data, dst->type,+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);+}++void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+ const ggml_tensor * src0 = dst->src[0]; // gradients of forward pass output+ const ggml_tensor * src1 = dst->src[1]; // src1 in forward pass++ GGML_TENSOR_BINARY_OP_LOCALS++ const float * src0_d = (const float *) src0->data;+ const int32_t * src1_d = (const int32_t *) src1->data;+ float * dst_d = (float *) dst->data;++ cudaStream_t stream = ctx.stream();++ GGML_ASSERT(src0->type == GGML_TYPE_F32);+ GGML_ASSERT(src1->type == GGML_TYPE_I32);+ GGML_ASSERT(dst->type == GGML_TYPE_F32);++ GGML_ASSERT(ggml_is_contiguous(src0));+ GGML_ASSERT(ggml_is_contiguous(src1));+ GGML_ASSERT(ggml_is_contiguous(dst));++ GGML_ASSERT(ne02*ne03 == 1);+ GGML_ASSERT(ne12*ne13 == 1);+ GGML_ASSERT(ne2*ne3 == 1);++ const dim3 block_dims(CUDA_GET_ROWS_BACK_BLOCK_SIZE, 1, 1);+ const int block_num_x = (ne00 + CUDA_GET_ROWS_BACK_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BACK_BLOCK_SIZE;+ const dim3 block_nums(block_num_x, MIN(ne1, (int64_t)UINT16_MAX), 1);++ k_get_rows_back_float<<<block_nums, block_dims, 0, stream>>>(src0_d, src1_d, dst_d, ne00, ne10, ne1);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cuhnew file mode 100644index 00000000..3c5bea5f--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/getrows.cuh@@ -0,0 +1,15 @@+#include "common.cuh"++#define CUDA_GET_ROWS_BLOCK_SIZE 256+#define CUDA_GET_ROWS_BACK_BLOCK_SIZE 256++void get_rows_cuda(+ const void * src0_d, ggml_type src0_type, const int32_t * src1_d, void * dst_d, ggml_type dst_type,+ int64_t ne00, size_t nb01, size_t nb02, size_t nb03,+ int64_t ne10, int64_t ne11, int64_t ne12, size_t nb10, size_t nb11, size_t nb12,+ size_t nb1, size_t nb2, size_t nb3,+ cudaStream_t stream);++void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_get_rows_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mma.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mma.cuhnew file mode 100644index 00000000..93a51aa0--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mma.cuh@@ -0,0 +1,1594 @@+#pragma once+// This file contains primitives that expose the tensor core PTX instructions for CUDA code.+// The primitives can be used in a similar way as the nvcuda::wmma interface but with a well-defined memory layout.+// The documentation for the PTX instructions can be found under:+// https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-multiply-accumulate-operation-using-mma-instruction+//+// Like with nvcuda::wmma there are three types of matrix tiles: A, B, and C with A @ B = C.+// A is a row-major matrix with shape M x K.+// B is a column-major matrix with shape K x N.+// C is a column-major matrix with shape M x N.+// A, B, and C are represented using the same fundamental data type: a row-major matrix with I rows and J columns.+// Note that J is measured in physical 32 bit elements instead of logical elements.+// The methods get_i and get_j can be used to get the physical 32 bit index of the lth element of a thread within a tile.+// All matrix tiles have ne physical 32 bit elements per warp.+//+// As described in the PTX documentation, all pointers for load_ldmatrix must be to shared memory and aligned to 16 bytes.+// The API in this file also assumes that the pointers for load_generic are aligned to 16 bytes, unaligned pointers are considered undefined behavior.++#include "common.cuh"++// On Volta each warp is doing 4 8x8 mma operations in parallel.+// The basic memory layout for a 32x8 output tile is to stack 4 input tiles in I direction and to mirror the B tile.+// However, the i indices in this file are by default permuted to simplify the index calculations.+// #define GGML_CUDA_MMA_NO_VOLTA_PERM++#if CUDART_VERSION >= 11080++static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {+ int ret = 0;++#ifdef TURING_MMA_AVAILABLE+ asm("movmatrix.sync.aligned.m8n8.trans.b16 %0, %1;"+ : "=r"(ret) : "r"(x));+#else+ GGML_UNUSED(x);+ NO_DEVICE_CODE;+#endif // defined(TURING_MMA_AVAILABLE)+ return ret;+}++#else++static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {+ // Imagine transposing row-major matrix to column-major matrix.+ const int src_i_low = 2 * (threadIdx.x % 4);+ const int src_i_high = src_i_low + 1;+ const int src_j = threadIdx.x / 4;++ const int src_laneid_low = src_i_low * 4 + src_j / 2;+ const int src_laneid_high = src_i_high * 4 + src_j / 2;++ const int shift_low = ((src_j + 0) % 2) * 16;+ const int shift_high = ((src_j + 1) % 2) * 16;++ const int ret_low = (__shfl_sync(0xFFFFFFFF, x, src_laneid_low, WARP_SIZE) >> shift_low) & 0x0000FFFF;+ const int ret_high = (__shfl_sync(0xFFFFFFFF, x, src_laneid_high, WARP_SIZE) << shift_high) & 0xFFFF0000;++ return ret_low | ret_high;+}++#endif // CUDART_VERSION >= 11080++static __device__ __forceinline__ half2 ggml_cuda_movmatrix(const half2 x) {+ half2 ret;+ *((int *) &ret) = ggml_cuda_movmatrix(*((const int *) &x));+ return ret;+}++namespace ggml_cuda_mma {++ // Some architectures like Volta or CDNA3 perform multiple matrix multiplications per warp in parallel,+ // effectively the warp is being split into subgroups of threads that each perform a single mma instruction.+ // In those cases the data can be split in different ways across the warp.+ enum data_layout {+ // By default the data uses the I direction as its major dimension and the J direction as its minor dimension.+ // For the A/C matrices this means I major == row major, J major == column major.+ // For the B matrix this means I major == column major, J major == row major.+ // MIRRORED == Each data value is held exactly once per thread subgroup.+ DATA_LAYOUT_I_MAJOR = 0, // Always used for Turing, Ampere, Ada Lovelace, consumer Blackwell, matrix A&B for RDNA4 and CDNA.+ DATA_LAYOUT_J_MAJOR = 10, // Matrix C for CDNA and RDNA4, int and float matrix C for RDNA3.+ DATA_LAYOUT_I_MAJOR_MIRRORED = 20, // Volta, matrix A&B for RDNA3.+ DATA_LAYOUT_J_MAJOR_MIRRORED = 30,+ DATA_LAYOUT_I_MAJOR_SCRAMBLED = 40, // Scrambled matrix C for faster transposition (RDNA4/CDNA), convert to float to unscramble.+ };+ // Implemented mma combinations are:+ // - (I_MAJOR, I_MAJOR) -> I_MAJOR+ // - (I_MAJOR, I_MAJOR_MIRRORED) -> I_MAJOR+ // - (I_MAJOR, J_MAJOR_MIRRORED) -> I_MAJOR++ static constexpr __device__ data_layout get_input_data_layout() {+#if defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE)+ return DATA_LAYOUT_I_MAJOR_MIRRORED;+#else+ return DATA_LAYOUT_I_MAJOR;+#endif // defined(RDNA3) || defined(VOLTA_MMA_AVAILABLE)+ }++ template <int I_, int J_, typename T, data_layout ds_=DATA_LAYOUT_I_MAJOR>+ struct tile {};++ template <int I_, int J_, typename T>+ struct tile<I_, J_, T, DATA_LAYOUT_I_MAJOR> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;++#if defined(AMD_MFMA_AVAILABLE)+ static constexpr int ne = I * J / 64;+ T x[ne] = {0};++ static constexpr __device__ bool supported() {+ if (I == 16 && J == 8) return true;+ if (I == 32 && J == 4) return true;+ if (I == 16 && J == 16) return true;+ if (I == 32 && J == 32) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 16 && J == 4) {+ return threadIdx.x % 16;+ } else if constexpr (I == 16 && J == 8) {+ return threadIdx.x % 16;+ } else if constexpr (I == 32 && J == 4) {+ return threadIdx.x % 32;+ } else if constexpr (I == 16 && J == 16) {+ return threadIdx.x % 16;+ } else if constexpr (I == 32 && J == 32) {+ return threadIdx.x % 32;+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 16 && J == 4) {+ return threadIdx.x / 16;+ } else if constexpr (I == 16 && J == 8) {+ return 2 * (threadIdx.x / 16) + l;+ } else if constexpr (I == 32 && J == 4) {+ return 2 * (threadIdx.x / 32) + l;+ } else if constexpr (I == 16 && J == 16) {+ return 4 * (threadIdx.x / 16) + l;+ } else if constexpr (I == 32 && J == 32) {+ return 4 * (threadIdx.x / 32) + 8 * (l / 4) + (l % 4);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#elif defined(VOLTA_MMA_AVAILABLE)+ static constexpr int ne = I * J / 32;+ T x[ne] = {0};++ static constexpr __device__ bool supported() {+ if (I == 32 && J == 8) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 32 && J == 8) {+#ifdef GGML_CUDA_MMA_NO_VOLTA_PERM+ return (((threadIdx.x % 16) / 4) * 8) + ((threadIdx.x / 16) * 4) + (l & 2) + (threadIdx.x % 2);+#else+ return (l & 2) + (threadIdx.x & ~2);+#endif // GGML_CUDA_MMA_NO_VOLTA_PERM+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 32 && J == 8) {+ return (threadIdx.x & 2) + (l & (4 + 1));+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#elif defined(AMD_WMMA_AVAILABLE)+ static constexpr int ne = I * J / 32;+ T x[ne] = {0};++ static constexpr __device__ bool supported() {+ if (I == 16 && J == 16) return true;+ if (I == 16 && J == 8) return true;+ if (I == 16 && J == 4) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (supported()) {+ return threadIdx.x % 16;+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 16 && J == 16) {+#if defined(RDNA3)+ if constexpr (std::is_same_v<T, float> || std::is_same_v<T, int>) {+ // matrix C+ return 2 * l + (threadIdx.x / 16);+ } else {+ // matrix A&B+ return l;+ }+#else+ // matrix C is the transposed matrix A&B on RDNA4+ return ne * (threadIdx.x / 16) + l;+#endif // defined(RDNA3)+ } else if constexpr (I == 16 && J == 8) {+ // mmq input for RDNA4+ return ne * (threadIdx.x / 16) + l;+ } else if constexpr (I == 16 && J == 4) {+ return ne * (threadIdx.x / 16) + l;+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#else+ static constexpr int ne = I * J / 32;+ T x[ne] = {0};++ static constexpr __device__ bool supported() {+ if (I == 8 && J == 4) return true;+ if (I == 8 && J == 8) return true;+ if (I == 16 && J == 8) return true;+ if (I == 16 && J == 16) return true;+ if (I == 32 && J == 8) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 8 && J == 4) {+ return threadIdx.x / 4;+ } else if constexpr (I == 8 && J == 8) {+ return threadIdx.x / 4;+ } else if constexpr (I == 16 && J == 8) {+ return ((l / 2) * 8) + (threadIdx.x / 4);+ } else if constexpr (I == 16 && J == 16) {+ return (((l / 2) % 2) * 8) + (threadIdx.x / 4);+ } else if constexpr (I == 32 && J == 8) {+ return tile<16, 8, T>::get_i(l); // Memory layout simply repeated with same pattern in i direction.+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 8 && J == 4) {+ return threadIdx.x % 4;+ } else if constexpr (I == 8 && J == 8) {+ return (l * 4) + (threadIdx.x % 4);+ } else if constexpr (I == 16 && J == 8) {+ return ((threadIdx.x % 4) * 2) + (l % 2);+ } else if constexpr (I == 16 && J == 16) {+ return ((l / 4) * 8) + ((threadIdx.x % 4) * 2) + (l % 2);+ } else if constexpr (I == 32 && J == 8) {+ return tile<16, 8, T>::get_j(l); // Memory layout simply repeated with same pattern in i direction.+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#endif // defined(GGML_USE_HIP)+ };++ template <int I_, int J_>+ struct tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;++#if defined(VOLTA_MMA_AVAILABLE)+ static constexpr int ne = I * J / WARP_SIZE;+ half2 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ if (I == 32 && J == 4) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 32 && J == 4) {+#ifdef GGML_CUDA_MMA_NO_VOLTA_PERM+ return (((threadIdx.x % 16) / 4) * 8) + ((threadIdx.x / 16) * 4) + (threadIdx.x % 4);+#else+ return threadIdx.x;+#endif // GGML_CUDA_MMA_NO_VOLTA_PERM+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 32 && J == 4) {+ return l;+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#elif defined(AMD_WMMA_AVAILABLE)+ static constexpr int ne = I * J / 32;+ half2 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ if (I == 16 && J == 8) return true;+ if (I == 16 && J == 16) return true;+ if (I == 32 && J == 8) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 16 && J == 8) {+ return threadIdx.x % 16;+ } else if constexpr (I == 16 && J == 16) {+ return threadIdx.x % 16;+ } else if constexpr (I == 32 && J == 8) {+ return (threadIdx.x % 16) * 2 + l / (ne/2);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 16 && J == 8) {+ return (threadIdx.x / 16) * ne + l;+ } else if constexpr (I == 16 && J == 16) {+#ifdef RDNA3+ return l*2 + (threadIdx.x / 16);+#else+ return (threadIdx.x / 16) * ne + l;+#endif // RDNA3+ } else if constexpr (I == 32 && J == 8) {+ return (threadIdx.x / 16) * (ne/2) + l % (ne/2);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#elif defined(AMD_MFMA_AVAILABLE)+ static constexpr int ne = I * J / 64;+ half2 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ if (I == 16 && J == 8) return true;+ if (I == 16 && J == 16) return true;+ if (I == 32 && J == 8) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 16 && J == 8) {+ return threadIdx.x % 16;+ } else if constexpr (I == 16 && J == 16) {+ return threadIdx.x % 16;+ } else if constexpr (I == 32 && J == 8) {+ return (threadIdx.x % 16) * 2 + l / (ne/2);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 16 && J == 8) {+ return (threadIdx.x / 16) * ne + l;+ } else if constexpr (I == 16 && J == 16) {+ return (threadIdx.x / 16) * ne + l;+ } else if constexpr (I == 32 && J == 8) {+ return (threadIdx.x / 16) * (ne/2) + l % (ne/2);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#else+ static constexpr int ne = I * J / WARP_SIZE;+ half2 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ if (I == 8 && J == 4) return true;+ if (I == 8 && J == 8) return true;+ if (I == 16 && J == 8) return true;+ if (I == 16 && J == 16) return true;+ if (I == 32 && J == 8) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 8 && J == 8) {+ return threadIdx.x / 4;+ } else if constexpr (I == 16 && J == 4) {+ return (l * 8) + (threadIdx.x / 4);+ } else if constexpr (I == 16 && J == 8) {+ return ((l % 2) * 8) + (threadIdx.x / 4);+ } else if constexpr (I == 32 && J == 8) {+ return ((l / 4) * 16) + ((l % 2) * 8) + (threadIdx.x / 4);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 8 && J == 8) {+ return (l * 4) + (threadIdx.x % 4);+ } else if constexpr (I == 16 && J == 4) {+ return threadIdx.x % 4;+ } else if constexpr (I == 16 && J == 8) {+ return ((l / 2) * 4) + (threadIdx.x % 4);+ } else if constexpr (I == 32 && J == 8) {+ return ((l & 2) * 2) + (threadIdx.x % 4);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#endif // defined(VOLTA_MMA_AVAILABLE)+ };++ template <int I_, int J_>+ struct tile<I_, J_, nv_bfloat162, DATA_LAYOUT_I_MAJOR> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR;++#if defined(AMD_WMMA_AVAILABLE)+ static constexpr int ne = tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::ne;+ nv_bfloat162 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::supported();+ }++ static __device__ __forceinline__ int get_i(const int l) {+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_i(l);+ }++ static __device__ __forceinline__ int get_j(const int l) {+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_j(l);+ }+#elif defined(AMD_MFMA_AVAILABLE)+ static constexpr int ne = tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::ne;+ nv_bfloat162 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::supported();+ }++ static __device__ __forceinline__ int get_i(const int l) {+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_i(l);+ }++ static __device__ __forceinline__ int get_j(const int l) {+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_j(l);+ }+#else+ static constexpr int ne = I * J / WARP_SIZE;+ nv_bfloat162 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ if (I == 8 && J == 8) return true;+ if (I == 16 && J == 4) return true;+ if (I == 16 && J == 8) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 8 && J == 8) {+ return threadIdx.x / 4;+ } else if constexpr (I == 16 && J == 4) {+ return (l * 8) + (threadIdx.x / 4);+ } else if constexpr (I == 16 && J == 8) {+ return ((l % 2) * 8) + (threadIdx.x / 4);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 8 && J == 8) {+ return (l * 4) + (threadIdx.x % 4);+ } else if constexpr (I == 16 && J == 4) {+ return threadIdx.x % 4;+ } else if constexpr (I == 16 && J == 8) {+ return ((l / 2) * 4) + (threadIdx.x % 4);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#endif // defined(AMD_WMMA_AVAILABLE)+ };++ template <int I_, int J_, typename T>+ struct tile<I_, J_, T, DATA_LAYOUT_J_MAJOR> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_J_MAJOR;++ static constexpr int ne = tile<I_, J_, T, DATA_LAYOUT_I_MAJOR>::ne;+ T x[ne] = {0};++ static constexpr __device__ bool supported() {+ return tile<I_, J_, T, DATA_LAYOUT_I_MAJOR>::supported();+ }++ static __device__ __forceinline__ int get_i(const int l) {+ return tile<I_, J_, T, DATA_LAYOUT_I_MAJOR>::get_j(l);+ }++ static __device__ __forceinline__ int get_j(const int l) {+ return tile<I_, J_, T, DATA_LAYOUT_I_MAJOR>::get_i(l);+ }+ };++ template <int I_, int J_, typename T>+ struct tile<I_, J_, T, DATA_LAYOUT_I_MAJOR_MIRRORED> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;++ // RDNA3+ static constexpr int ne = I * J / 32 * 2;++ T x[ne] = {0};++ static constexpr __device__ bool supported() {+ if (I == 16 && J == 16) return true;+ if (I == 16 && J == 8) return true;+ if (I == 16 && J == 4) return true;+ if (I == 32 && J == 8) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 16) {+ return threadIdx.x % 16;+ } else if constexpr (I == 32) {+ return (threadIdx.x % 16) * 2 + l / (ne/2);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 16) {+ return l;+ } else if constexpr (I == 32) {+ return l % (ne/2);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+ };++ template <int I_, int J_>+ struct tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;+#if defined(RDNA3)+ static constexpr int ne = tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::ne;++ half2 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::supported();+ }++ static __device__ __forceinline__ int get_i(const int l) {+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::get_i(l);+ }++ static __device__ __forceinline__ int get_j(const int l) {+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::get_j(l);+ }+#else // Volta+ static constexpr int ne = I * J / (WARP_SIZE/4);++ half2 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ if (I == 8 && J == 4) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int /*l*/) {+ if constexpr (I == 8 && J == 4) {+ return ((threadIdx.x / 16) * 4) + (threadIdx.x % 4);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 8 && J == 4) {+ return l;+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+#endif // defined(RDNA3)+ };++ template <int I_, int J_>+ struct tile<I_, J_, nv_bfloat162, DATA_LAYOUT_I_MAJOR_MIRRORED> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_MIRRORED;+ static constexpr int ne = tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::ne;++ nv_bfloat162 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::supported();+ }++ static __device__ __forceinline__ int get_i(const int l) {+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::get_i(l);+ }++ static __device__ __forceinline__ int get_j(const int l) {+ return tile<I_, J_, float, DATA_LAYOUT_I_MAJOR_MIRRORED>::get_j(l);+ }+ };++ template <int I_, int J_>+ struct tile<I_, J_, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_J_MAJOR_MIRRORED;+ static constexpr int ne = I * J / (WARP_SIZE/4);++ half2 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ if (I == 8 && J == 4) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ if constexpr (I == 8 && J == 4) {+ return ((l / 2) * 4) + (threadIdx.x % 4);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }++ static __device__ __forceinline__ int get_j(const int l) {+ if constexpr (I == 8 && J == 4) {+ return ((threadIdx.x / 16) * 2) + (l % 2);+ } else {+ NO_DEVICE_CODE;+ return -1;+ }+ }+ };++ template <int I_, int J_>+ struct tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> {+ static constexpr int I = I_;+ static constexpr int J = J_;+ static constexpr data_layout dl = DATA_LAYOUT_I_MAJOR_SCRAMBLED;++ static constexpr int ne = I * J / ggml_cuda_get_physical_warp_size();+ half2 x[ne] = {{0.0f, 0.0f}};++ static constexpr __device__ bool supported() {+ if (I == 16 && J == 16) return true;+ return false;+ }++ static __device__ __forceinline__ int get_i(const int l) {+ return tile<I_, J_, half2, DATA_LAYOUT_I_MAJOR>::get_i(l);+ }+ };++ static __device__ __forceinline__ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> unscramble(const tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & t) {+#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))+ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> ret;+#pragma unroll+ for (int l0 = 0; l0 < t.ne/2; ++l0) {+ ret.x[2*l0 + 0] = __lows2half2(t.x[l0], t.x[l0 + t.ne/2]);+ ret.x[2*l0 + 1] = __highs2half2(t.x[l0], t.x[l0 + t.ne/2]);+ }+ return ret;+#else+ NO_DEVICE_CODE;+ GGML_UNUSED(t);+#endif // defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))+ }++#if defined(TURING_MMA_AVAILABLE)+ template <int I, int J>+ static __device__ __forceinline__ tile<I, J/2, half2> get_half2(const tile<I, J, float> & tile_float) {+ tile<I, J/2, half2> ret;+#pragma unroll+ for (int l0 = 0; l0 < tile_float.ne; l0 += 2) {+ ret.x[l0/2] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);+ }+ return ret;+ }++ static __device__ __forceinline__ tile<8, 8, half2> get_transposed(const tile<16, 4, half2> & t) {+ tile<8, 8, half2> ret;+ ret.x[0] = ggml_cuda_movmatrix(t.x[0]);+ ret.x[1] = ggml_cuda_movmatrix(t.x[1]);++ return ret;+ }+#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)+ static __device__ __forceinline__ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> get_half2(+ const tile<16, 16, float, DATA_LAYOUT_I_MAJOR> & tile_float) {+ tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> ret;+#pragma unroll+ for (int l = 0; l < tile_float.ne; ++l) {+ float tmp[2];+ int i = threadIdx.x / 16;+ tmp[i] = tile_float.x[l];+ i ^= 1;+ tmp[i] = __shfl_xor_sync(0xFFFFFFFF, tile_float.x[l], 16, WARP_SIZE);+ ret.x[l] = make_half2(tmp[0], tmp[1]);+ }+ return ret;+ }+#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)+ template <int I, int J>+ static __device__ __forceinline__ tile<I, J/2, half2> get_half2(const tile<I, J, float> & tile_float) {+ tile<I, J/2, half2> ret;+#pragma unroll+ for (int l0 = 0; l0 < tile_float.ne; l0 += 2) {+ ret.x[l0/2] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);+ }+ return ret;+ }++ static __device__ __forceinline__ tile<8, 8, half2> get_transposed(const tile<16, 4, half2> & t) {+ NO_DEVICE_CODE;+ return tile<8, 8, half2>{};+ }+#else // Volta+ template <int I, int J>+ static __device__ __forceinline__ tile<I, J/2, half2> get_half2(const tile<I, J, float> & tile_float) {+ tile<I, J/2, half2> ret;+#pragma unroll+ for (int l0 = 0; l0 < tile_float.ne; l0 += 4) {+ ret.x[l0/2 + 0] = make_half2(tile_float.x[l0 + 0], tile_float.x[l0 + 1]);+ ret.x[l0/2 + 1] = make_half2(tile_float.x[l0 + 2], tile_float.x[l0 + 3]);++ // On Volta FP16 and FP32 tiles have a different memory layout,+ // for the conversion threads with an offset of 2 need to exchange half their values:+ ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)] = __shfl_xor_sync(+ 0xFFFFFFFF, ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)], 2, WARP_SIZE);+ }+ return ret;+ }+#endif // defined(TURING_MMA_AVAILABLE)++ template <int I, int J, typename T, data_layout dl>+ static __device__ __forceinline__ void load_generic(tile<I, J, T, dl> & t, const T * __restrict__ xs0, const int stride) {+#pragma unroll+ for (int l = 0; l < t.ne; ++l) {+ t.x[l] = xs0[t.get_i(l)*stride + t.get_j(l)];+ }+ }++ template <int stride, typename T>+ static __device__ __forceinline__ uint32_t swizzle(const uint32_t offset, const uint32_t i) {+ static_assert(sizeof(T) <= 4, "unsupported type size");+ constexpr int stride_bytes = stride*sizeof(T);+ static_assert(stride_bytes % 16 == 0, "bad stride");+ constexpr uint32_t shift = sizeof(T) == 1 ? 4 : (sizeof(T) == 2 ? 3 : 2);+ if (stride_bytes % 32 != 0) {+ return offset; // Equivalent to padding with 16 bytes.+ }+ if (stride_bytes % 64 != 0) {+ return offset ^ (((i / 4) % 2) << shift);+ }+ if (stride_bytes % 128 != 0) {+ return offset ^ (((i / 2) % 4) << shift);+ }+ return offset ^ ((i % 8) << shift);+ }++ template <int stride, typename T>+ static __device__ __forceinline__ T * swizzle(T * ptr, const uint32_t offset, const uint32_t i) {+ return ptr + swizzle<stride, T>(offset, i);+ }++ template <typename T>+ static __device__ __forceinline__ void load_ldmatrix(+ tile<8, 8, T> & t, const T * __restrict__ xs0, const int stride) {+#ifdef TURING_MMA_AVAILABLE+ int * xi = (int *) t.x;+ const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + ((threadIdx.x / t.I) * (t.J / 2)) % t.J;+ asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];"+ : "=r"(xi[0]), "=r"(xi[1])+ : "l"(xs));+#else+ GGML_UNUSED_VARS(t, xs0, stride);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ template <typename T, data_layout dl>+ static __device__ __forceinline__ void load_ldmatrix(+ tile<16, 4, T, dl> & t, const T * __restrict__ xs0, const int stride) {+#ifdef TURING_MMA_AVAILABLE+ int * xi = (int *) t.x;+ const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride;+ asm volatile("ldmatrix.sync.aligned.m8n8.x2.b16 {%0, %1}, [%2];"+ : "=r"(xi[0]), "=r"(xi[1])+ : "l"(xs));+#elif defined(AMD_WMMA_AVAILABLE)+#ifdef RDNA3+ static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");+ static_assert(sizeof(t.x) == 16, "bad ne");+ ggml_cuda_memcpy_1<8>(t.x + 0, xs0 + t.get_i(0)*stride + 0);+ ggml_cuda_memcpy_1<8>(t.x + 2, xs0 + t.get_i(0)*stride + 2);+#else+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");+ static_assert(sizeof(t.x) == 8, "bad ne");+ ggml_cuda_memcpy_1<8>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0));+#endif // RDNA3+#elif defined(AMD_MFMA_AVAILABLE)+ static_assert(sizeof(t.x) == 4, "bad ne");+ ggml_cuda_memcpy_1<4>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0));+#else+ GGML_UNUSED_VARS(t, xs0, stride);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ template <typename T, data_layout dl>+ static __device__ __forceinline__ void load_ldmatrix(+ tile<16, 8, T, dl> & t, const T * __restrict__ xs0, const int stride) {+#if defined(TURING_MMA_AVAILABLE)+ int * xi = (int * ) t.x;+ const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2);+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];"+ : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3])+ : "l"(xs));+#elif defined(VOLTA_MMA_AVAILABLE)+ ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 0, xs0 + t.get_i(0)*stride + 0);+ ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + 4, xs0 + t.get_i(4)*stride + 4);+#elif defined(AMD_WMMA_AVAILABLE)+#ifdef RDNA3+ static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");+ static_assert(sizeof(t.x) == 32, "bad ne");+ ggml_cuda_memcpy_1<16>(t.x + 0, xs0 + t.get_i(0)*stride + 0);+ ggml_cuda_memcpy_1<16>(t.x + 4, xs0 + t.get_i(0)*stride + 4);+#else+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");+ static_assert(sizeof(t.x) == 16, "bad ne");+ ggml_cuda_memcpy_1<16>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0));+#endif // RDNA3+#elif defined(AMD_MFMA_AVAILABLE)+ static_assert(sizeof(t.x) == 8, "bad ne");+ ggml_cuda_memcpy_1<8>(t.x, xs0 + t.get_i(0)*stride + t.get_j(0));+#else+ GGML_UNUSED_VARS(t, xs0, stride);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ static __device__ __forceinline__ void load_ldmatrix(+ tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) {+ ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride);+ }++ static __device__ __forceinline__ void load_ldmatrix(+ tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int stride) {+#ifdef VOLTA_MMA_AVAILABLE+#pragma unroll+ for (int l0 = 0; l0 < t.ne; l0 += 2) {+ ggml_cuda_memcpy_1<2*sizeof(half2)>(t.x + l0, xs0 + t.get_i(l0)*stride + t.get_j(l0));+ }+#else+ GGML_UNUSED_VARS(t, xs0, stride);+ NO_DEVICE_CODE;+#endif // VOLTA_MMA_AVAILABLE+ }++ static __device__ __forceinline__ void load_ldmatrix(+ tile<32, 4, half2> & t, const half2 * __restrict__ xs0, const int stride) {+#if defined(VOLTA_MMA_AVAILABLE)+ ggml_cuda_memcpy_1<4*sizeof(half2)>(t.x, xs0 + t.get_i(0)*stride);+#else+ GGML_UNUSED_VARS(t, xs0, stride);+ NO_DEVICE_CODE;+#endif // defined(VOLTA_MMA_AVAILABLE)+ }++ template <int I, typename T, data_layout dl>+ static __device__ __forceinline__ void load_ldmatrix_trans(+ tile<I, 8, T, dl> & t, const T * __restrict__ xs0, const int stride) {+#ifdef TURING_MMA_AVAILABLE+ static_assert(I == 16, "bad tile width");+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");+ int * xi = (int *) t.x;+ const int * xs = (const int *) xs0 + (threadIdx.x % t.I) * stride + (threadIdx.x / t.I) * (t.J / 2);+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];"+ : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3])+ : "l"(xs));+#elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");+ if constexpr (I == 32) {+#pragma unroll+ for (int l0 = 0; l0 < t.ne/2; ++l0) {+ const half2 tmp0 = xs0[(2*t.get_j(l0) + 0)*stride + t.get_i(l0)/2];+ const half2 tmp1 = xs0[(2*t.get_j(l0) + 1)*stride + t.get_i(l0)/2];++ t.x[l0] = __lows2half2(tmp0, tmp1);+ t.x[l0 + t.ne/2] = __highs2half2(tmp0, tmp1);+ }+ } else {+ half * xh = (half *) t.x;+#pragma unroll+ for (int l = 0; l < t.ne; ++l) {+ xh[2*l + 0] = ((const half *) xs0)[(2*t.get_j(l) + 0)*(2*stride) + t.get_i(l)];+ xh[2*l + 1] = ((const half *) xs0)[(2*t.get_j(l) + 1)*(2*stride) + t.get_i(l)];+ }+ }+#else+ GGML_UNUSED_VARS(t, xs0, stride);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ template <int stride, int I, int J, typename T, data_layout dl>+ static __device__ __forceinline__ void load_ldmatrix_swizzled(+ tile<I, J, T, dl> & t, const T * __restrict__ xs0, const int offset) {+#if defined(TURING_MMA_AVAILABLE)+ static_assert(I == 16, "bad tile width");+ static_assert(J == 8, "bad tile height");+ const int i = threadIdx.x % t.I;+ const int j = (threadIdx.x / t.I) * (t.J / 2);+ int offset_ij = offset + i * stride + j;+ offset_ij = swizzle<stride, T>(offset_ij, i);+ int * xi = (int *) t.x;+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.b16 {%0, %1, %2, %3}, [%4];"+ : "=r"(xi[0]), "=r"(xi[1]), "=r"(xi[2]), "=r"(xi[3])+ : "l"(xs0 + offset_ij));+#elif defined(VOLTA_MMA_AVAILABLE)+#pragma unroll+ for (int o = 0; o < t.ne; o += 4) {+ const int offset_ij = offset + t.get_i(o) * stride + o;+ ggml_cuda_memcpy_1<4*sizeof(T)>(t.x + o, swizzle<stride>(xs0, offset_ij, t.get_i(o)));+ }+#elif defined(AMD_WMMA_AVAILABLE)+#ifdef RDNA3+ static_assert(dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");+ static_assert(sizeof(t.x) == 32, "bad ne");+ static_assert(I == 16, "bad tile width");+ static_assert(J == 8, "bad tile height");+#pragma unroll+ for (int o = 0; o < 8; o += 4) {+ const int offset_ij = offset + t.get_i(0) * stride + o;+ ggml_cuda_memcpy_1<16>(t.x + o, swizzle<stride>(xs0, offset_ij, t.get_i(0)));+ }+#else+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");+ static_assert(sizeof(t.x) == 16, "bad ne");+ const int offset_ij = offset + t.get_i(0)*stride + t.get_j(0);+ ggml_cuda_memcpy_1<16>(t.x, swizzle<stride>(xs0, offset_ij, t.get_i(0)));+#endif // RDNA3+#elif defined(AMD_MFMA_AVAILABLE)+ static_assert(sizeof(t.x) == 8, "bad ne");+ const int offset_ij = offset + t.get_i(0)*stride + t.get_j(0);+ ggml_cuda_memcpy_1<8>(t.x, swizzle<stride>(xs0, offset_ij, t.get_i(0)));+#else+ GGML_UNUSED_VARS(t, xs0, offset);+ NO_DEVICE_CODE;+#endif // defined(TURING_MMA_AVAILABLE)+ }++ template <int stride>+ static __device__ __forceinline__ void load_ldmatrix_swizzled(+ tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & t, const half2 * __restrict__ xs0, const int offset) {+#ifdef VOLTA_MMA_AVAILABLE+#pragma unroll+ for (int l0 = 0; l0 < t.ne; l0 += 2) {+ const int offset_ij = offset + t.get_i(l0)*stride + t.get_j(l0);+ ggml_cuda_memcpy_1<2*sizeof(half2)>(t.x + l0, swizzle<stride>(xs0, offset_ij, t.get_i(l0)));+ }+#else+ GGML_UNUSED_VARS(t, xs0, offset);+ NO_DEVICE_CODE;+#endif // VOLTA_MMA_AVAILABLE+ }++ template <int stride, int I, typename T, data_layout dl>+ static __device__ __forceinline__ void load_ldmatrix_trans_swizzled(+ tile<I, 8, T, dl> & t, const T * __restrict__ xs0, const int offset) {+#if defined(TURING_MMA_AVAILABLE)+ static_assert(I == 16, "bad tile width");+ static_assert(dl == DATA_LAYOUT_I_MAJOR, "bad data layout");+ const int i = threadIdx.x % t.I;+ const int j = (threadIdx.x / t.I) * (t.J / 2);+ int offset_ij = offset + i * stride + j;+ offset_ij = swizzle<stride, T>(offset_ij, i);+ int * xi = (int *) t.x;+ asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.b16 {%0, %1, %2, %3}, [%4];"+ : "=r"(xi[0]), "=r"(xi[2]), "=r"(xi[1]), "=r"(xi[3])+ : "l"(xs0 + offset_ij));+#elif defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ static_assert(dl == DATA_LAYOUT_I_MAJOR || dl == DATA_LAYOUT_I_MAJOR_MIRRORED, "bad data layout");+ if constexpr (I == 32) {+#pragma unroll+ for (int l0 = 0; l0 < t.ne/2; ++l0) {+ half2 tmp[2];+#pragma unroll+ for (int o = 0; o < 2; ++o) {+ const int j = 2*t.get_j(l0) + o;+ int offset_ij = offset + j*stride + t.get_i(l0)/2;+ offset_ij = swizzle<stride, T>(offset_ij, j);+ tmp[o] = xs0[offset_ij];+ }++ t.x[l0] = __lows2half2(tmp[0], tmp[1]);+ t.x[l0 + t.ne/2] = __highs2half2(tmp[0], tmp[1]);+ }+ } else {+ half * xh = (half *) t.x;+#pragma unroll+ for (int l = 0; l < t.ne; ++l) {+#pragma unroll+ for (int o = 0; o < 2; ++o) {+ const int j = 2*t.get_j(l) + o;+ xh[2*l + o] = ((const half *) xs0)[swizzle<2*stride, half>(2*offset + j*(2*stride) + t.get_i(l), j)];+ }+ }+ }+#else+ GGML_UNUSED_VARS(t, xs0, offset);+ NO_DEVICE_CODE;+#endif // defined(TURING_MMA_AVAILABLE)+ }++ static __device__ __forceinline__ void mma(+ tile<16, 8, int> & D, const tile<16, 4, int> & A, const tile<8, 4, int> & B) {+#ifdef TURING_MMA_AVAILABLE+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+ asm("mma.sync.aligned.m16n8k16.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"+ : "+r"(D.x[0]), "+r"(D.x[1]), "+r"(D.x[2]), "+r"(D.x[3])+ : "r"(A.x[0]), "r"(A.x[1]), "r"(B.x[0]));+#else+ // On Turing m16n8k16 mma is not available, use 2x m8n8k16 mma instead:+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"+ : "+r"(D.x[0]), "+r"(D.x[1])+ : "r"(A.x[0]), "r"(B.x[0]));+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"+ : "+r"(D.x[2]), "+r"(D.x[3])+ : "r"(A.x[1]), "r"(B.x[0]));+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ static __device__ __forceinline__ void mma(+ tile<16, 8, int> & D, const tile<16, 8, int> & A, const tile<8, 8, int> & B) {+#ifdef TURING_MMA_AVAILABLE+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+ asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"+ : "+r"(D.x[0]), "+r"(D.x[1]), "+r"(D.x[2]), "+r"(D.x[3])+ : "r"(A.x[0]), "r"(A.x[1]), "r"(A.x[2]), "r"(A.x[3]), "r"(B.x[0]), "r"(B.x[1]));+#else+ // On Turing m16n8k32 mma is not available, use 4x m8n8k16 mma instead:+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"+ : "+r"(D.x[0]), "+r"(D.x[1])+ : "r"(A.x[0]), "r"(B.x[0]));+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"+ : "+r"(D.x[2]), "+r"(D.x[3])+ : "r"(A.x[1]), "r"(B.x[0]));+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"+ : "+r"(D.x[0]), "+r"(D.x[1])+ : "r"(A.x[2]), "r"(B.x[1]));+ asm("mma.sync.aligned.m8n8k16.row.col.s32.s8.s8.s32 {%0, %1}, {%2}, {%3}, {%0, %1};"+ : "+r"(D.x[2]), "+r"(D.x[3])+ : "r"(A.x[3]), "r"(B.x[1]));+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ static __device__ __forceinline__ void mma(+ tile<16, 4, half2> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) {+#ifdef TURING_MMA_AVAILABLE+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ int * Dxi = (int *) D.x;+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+ asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};"+ : "+r"(Dxi[0]), "+r"(Dxi[1])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]));+#else+ // On Turing m16n8k16 mma is not available, use 2x m8n8k8 mma instead:+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"+ : "+r"(Dxi[0]), "+r"(Dxi[1])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]));+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"+ : "+r"(Dxi[0]), "+r"(Dxi[1])+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]));+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ static __device__ __forceinline__ void mma(+ tile<16, 8, half2> & D, const tile<16, 8, half2> & A, const tile<16, 8, half2> & B) {+#ifdef TURING_MMA_AVAILABLE+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ int * Dxi = (int *) D.x;+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+ asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};"+ : "+r"(Dxi[0]), "+r"(Dxi[1])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[2]));+ asm("mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3, %4, %5}, {%6, %7}, {%0, %1};"+ : "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]), "r"(Bxi[3]));+#else+ // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead:+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"+ : "+r"(Dxi[0]), "+r"(Dxi[1])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]));+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"+ : "+r"(Dxi[0]), "+r"(Dxi[1])+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]));+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"+ : "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[1]));+ asm("mma.sync.aligned.m16n8k8.row.col.f16.f16.f16.f16 {%0, %1}, {%2, %3}, {%4}, {%0, %1};"+ : "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3]));+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+#elif defined(AMD_WMMA_AVAILABLE)+#if defined(RDNA4)+ using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;+ halfx8_t& acc_frag = reinterpret_cast<halfx8_t&>(D.x[0]);+ const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]);+ const halfx8_t& b_frag = reinterpret_cast<const halfx8_t&>(B.x[0]);+ acc_frag = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag);+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // defined(RDNA4)+#elif defined(AMD_MFMA_AVAILABLE)+ // MFMA: FP16 input, FP32 accumulate, convert back to half2.+ using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;+ using floatx4_t = __attribute__((ext_vector_type(4))) float;++ // Convert existing half2 accumulator to float for MFMA:+ floatx4_t acc_f32;+ {+ const halfx4_t acc_h = reinterpret_cast<const halfx4_t&>(D.x[0]);+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ acc_f32[i] = (float)acc_h[i];+ }+ }++ const halfx4_t& a_frag = reinterpret_cast<const halfx4_t&>(A.x[0]);+ const halfx4_t& b_frag = reinterpret_cast<const halfx4_t&>(B.x[0]);+ acc_f32 = __builtin_amdgcn_mfma_f32_16x16x16f16(a_frag, b_frag, acc_f32, 0, 0, 0);++ // Convert back to half2:+ {+ halfx4_t result_h;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ result_h[i] = (_Float16)acc_f32[i];+ }+ reinterpret_cast<halfx4_t&>(D.x[0]) = result_h;+ }+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ static __device__ __forceinline__ void mma(+ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR_SCRAMBLED> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR> & A,+ const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR> & B) {+#if defined(AMD_MFMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA4))+ tile<16, 8, half2> * D16 = (tile<16, 8, half2> *) &D;+ const tile<16, 8, half2> * A16 = (const tile<16, 8, half2> *) &A;+ mma(D16[0], A16[0], B);+ mma(D16[1], A16[1], B);+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) && defined(RDNA4)+ }++ template <data_layout dl_ab, data_layout dl_d>+ static __device__ __forceinline__ void mma(+ tile<16, 8, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<8, 8, float, dl_ab> & B) {+#ifdef AMPERE_MMA_AVAILABLE+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ int * Dxi = (int *) D.x;+ asm("mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]));+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // AMPERE_MMA_AVAILABLE+ }++ template <data_layout dl_ab, data_layout dl_d>+ static __device__ __forceinline__ void mma(+ tile<16, 16, float, dl_d> & D, const tile<16, 8, float, dl_ab> & A, const tile<16, 8, float, dl_ab> & B) {+#ifdef AMD_MFMA_AVAILABLE+ using floatx4_t = __attribute__((ext_vector_type(4))) float;+ floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]);+#if defined(CDNA3)+ using floatx2_t = __attribute__((ext_vector_type(2))) float;+ const floatx2_t& a_frag = reinterpret_cast<const floatx2_t&>(A.x[0]);+ const floatx2_t& b_frag = reinterpret_cast<const floatx2_t&>(B.x[0]);+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x8_xf32(a_frag, b_frag, acc_frag, 0, 0, 0);+#elif defined(CDNA4) || defined(CDNA2) || defined(CDNA1)+ // CDNA4 (gfx950) does not support xf32 MFMA, use f32 path like CDNA2/CDNA1+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x4f32(A.x[i], B.x[i], acc_frag, 0, 0, 0);+ }+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // defined(CDNA3)+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE+ }++ template <ggml_type type>+ static __device__ __forceinline__ void mma_block_scaled_fp4(tile<16, 8, float> & D,+ const tile<16, 8, int> & A,+ const tile<8, 8, int> & B,+ uint32_t a_scale,+ uint32_t b_scale) {+#ifdef BLACKWELL_MMA_AVAILABLE+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ float * Dxi = (float *) D.x;++ if constexpr (type == GGML_TYPE_MXFP4) {+ asm volatile(+ "mma.sync.aligned.kind::mxf4.block_scale.scale_vec::2X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue8m0 "+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, "+ "%10, {0, 0}, %11, {0, 0};"+ : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale));+ } else {+ asm volatile(+ "mma.sync.aligned.kind::mxf4nvf4.block_scale.scale_vec::4X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue4m3 "+ "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3}, "+ "%10, {0, 0}, %11, {0, 0};"+ : "+f"(Dxi[0]), "+f"(Dxi[1]), "+f"(Dxi[2]), "+f"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]), "r"(a_scale), "r"(b_scale));+ }+#else+ GGML_UNUSED_VARS(D, A, B, a_scale, b_scale);+#endif // BLACKWELL_MMA_AVAILABLE+ }++ static __device__ __forceinline__ void mma(+ tile<16, 8, float> & D, const tile<16, 8, half2> & A, const tile<8, 8, half2> & B) {+#ifdef TURING_MMA_AVAILABLE+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ int * Dxi = (int *) D.x;+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+ asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]));+#else+ // On Turing m16n8k16 mma is not available, use 2x m8n8k8 mma instead:+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]));+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]));+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ static __device__ __forceinline__ void mma(+ tile<16, 8, float> & D, const tile<16, 8, nv_bfloat162> & A, const tile<8, 8, nv_bfloat162> & B) {+#ifdef AMPERE_MMA_AVAILABLE+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ int * Dxi = (int *) D.x;+ asm("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[1]));+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // AMPERE_MMA_AVAILABLE+ }++ template <data_layout dl_ab, data_layout dl_d>+ static __device__ __forceinline__ void mma(+ tile<16, 16, float, dl_d> & D, const tile<16, 8, half2, dl_ab> & A, const tile<16, 8, half2, dl_ab> & B) {+#ifdef TURING_MMA_AVAILABLE+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ int * Dxi = (int *) D.x;+#if __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+ asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[0]), "r"(Bxi[2]));+ asm("mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};"+ : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[1]), "r"(Bxi[3]));+#else+ // On Turing m16n8k16 mma is not available, use 4x m8n8k8 mma instead:+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]));+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]));+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"+ : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[1]));+ asm("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 {%0, %1, %2, %3}, {%4, %5}, {%6}, {%0, %1, %2, %3};"+ : "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[3]));+#endif // __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE+#elif defined(AMD_WMMA_AVAILABLE)+#if defined(RDNA4)+ using halfx8_t = __attribute__((ext_vector_type(8))) _Float16;+ using floatx8_t = __attribute__((ext_vector_type(8))) float;+ floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);+ const halfx8_t& a_frag = reinterpret_cast<const halfx8_t&>(A.x[0]);+ const halfx8_t& b_frag = reinterpret_cast<const halfx8_t&>(B.x[0]);+ acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a_frag, b_frag, acc_frag);+#elif defined(RDNA3)+ using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;+ using floatx8_t = __attribute__((ext_vector_type(8))) float;+ floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);+ const halfx16_t& a_frag = reinterpret_cast<const halfx16_t&>(A.x[0]);+ const halfx16_t& b_frag = reinterpret_cast<const halfx16_t&>(B.x[0]);+ acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a_frag, b_frag, acc_frag);+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // RDNA4+#elif defined(AMD_MFMA_AVAILABLE)+ using halfx4_t = __attribute__((ext_vector_type(4))) _Float16;+ using floatx4_t = __attribute__((ext_vector_type(4))) float;+ floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]);+ const halfx4_t& a_frag = reinterpret_cast<const halfx4_t&>(A.x[0]);+ const halfx4_t& b_frag = reinterpret_cast<const halfx4_t&>(B.x[0]);+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x16f16(a_frag, b_frag, acc_frag, 0, 0, 0);+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ template <data_layout dl_ab, data_layout dl_d>+ static __device__ __forceinline__ void mma(+ tile<16, 16, float, dl_d> & D, const tile<16, 8, nv_bfloat162, dl_ab> & A, const tile<16, 8, nv_bfloat162, dl_ab> & B) {+#if defined(AMD_WMMA_AVAILABLE)+#if defined(RDNA4)+ using bf16x8_t = __attribute__((ext_vector_type(8))) __bf16;+ using floatx8_t = __attribute__((ext_vector_type(8))) float;+ floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);+ const bf16x8_t& a_frag = reinterpret_cast<const bf16x8_t&>(A.x[0]);+ const bf16x8_t& b_frag = reinterpret_cast<const bf16x8_t&>(B.x[0]);+ acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12(a_frag, b_frag, acc_frag);+#elif defined(RDNA3)+ using bf16x16_t = __attribute__((ext_vector_type(16))) __bf16;+ using floatx8_t = __attribute__((ext_vector_type(8))) float;+ floatx8_t& acc_frag = reinterpret_cast<floatx8_t&>(D.x[0]);+ const bf16x16_t& a_frag = reinterpret_cast<const bf16x16_t&>(A.x[0]);+ const bf16x16_t& b_frag = reinterpret_cast<const bf16x16_t&>(B.x[0]);+ acc_frag = __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32(a_frag, b_frag, acc_frag);+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // defined(RDNA4)+#elif defined(AMD_MFMA_AVAILABLE)+ using floatx4_t = __attribute__((ext_vector_type(4))) float;+ floatx4_t& acc_frag = reinterpret_cast<floatx4_t&>(D.x[0]);+#if defined(CDNA4) || defined(CDNA3) || defined(CDNA2)+ using bf16x4_t = __attribute__((ext_vector_type(4))) __bf16;+ const bf16x4_t& a_frag = reinterpret_cast<const bf16x4_t&>(A.x[0]);+ const bf16x4_t& b_frag = reinterpret_cast<const bf16x4_t&>(B.x[0]);+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x16bf16_1k(a_frag, b_frag, acc_frag, 0, 0, 0);+#elif defined(CDNA1)+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ using bf16x2_t = __attribute__((ext_vector_type(2))) __bf16;+ const bf16x2_t& a_frag = reinterpret_cast<const bf16x2_t&>(A.x[i]);+ const bf16x2_t& b_frag = reinterpret_cast<const bf16x2_t&>(B.x[i]);+ acc_frag = __builtin_amdgcn_mfma_f32_16x16x8bf16(a_frag, b_frag, acc_frag, 0, 0, 0);+ }+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // defined(CDNA3) || defined(CDNA2)+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // defined(AMD_WMMA_AVAILABLE)+ }++ template <data_layout dl_d, data_layout dl_ab>+ static __device__ __forceinline__ void mma(+ tile<16, 16, int, dl_d> & D, const tile<16, 8, int, dl_ab> & A, const tile<16, 8, int, dl_ab> & B) {+#if defined(AMD_MFMA_AVAILABLE)+ using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;+ int32x4_t * acc = (int32x4_t *) D.x;+#if defined(CDNA4) || defined(CDNA3)+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(((int64_t *) A.x)[0], ((int64_t *) B.x)[0], acc[0], 0, 0, 0);+#elif defined(CDNA2) || defined(CDNA1)+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], B.x[0], acc[0], 0, 0, 0);+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[1], B.x[1], acc[0], 0, 0, 0);+#endif // defined(CDNA4) || defined(CDNA3)+#elif defined(AMD_WMMA_AVAILABLE)+ using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;+ int32x8_t * acc = (int32x8_t *) D.x;+#if defined(RDNA4)+ using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;+ int32x2_t * a_vec = (int32x2_t *) A.x;+ int32x2_t * b_vec = (int32x2_t *) B.x;+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], true);+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[1], true, b_vec[1], acc[0], true);+#elif defined(RDNA3)+ using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;+ int32x4_t * a_vec = (int32x4_t *) A.x;+ int32x4_t * b_vec = (int32x4_t *) B.x;+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], true);+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[1], true, b_vec[1], acc[0], true);+#endif // RDNA4+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE+ }++ static __device__ __forceinline__ void mma(+ tile<32, 32, int> & D, const tile<32, 4, int> & A, const tile<32, 4, int> & B) {+#if defined(AMD_MFMA_AVAILABLE)+ using int32x16_t = __attribute__((__vector_size__(16 * sizeof(int)))) int;+ int32x16_t * acc = (int32x16_t *) D.x;+#if defined(CDNA4) || defined(CDNA3)+ acc[0] = __builtin_amdgcn_mfma_i32_32x32x16_i8(((int64_t *) A.x)[0], ((int64_t *) B.x)[0], acc[0], 0, 0, 0);+#elif defined(CDNA2) || defined(CDNA1)+ acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[0], B.x[0], acc[0], 0, 0, 0);+ acc[0] = __builtin_amdgcn_mfma_i32_32x32x8i8(A.x[1], B.x[1], acc[0], 0, 0, 0);+#endif // defined(CDNA4) || defined(CDNA3)++#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE+ }++ template <typename T1, typename T2, int J, int K>+ static __device__ __forceinline__ void mma(+ tile<32, J, T1> & D, const tile<32, K, T2> & A, const tile<J, K, T2> & B) {+ tile <16, J, T1> * D16 = reinterpret_cast< tile<16, J, T1> *>(&D);+ const tile<16, K, T2> * A16 = reinterpret_cast<const tile<16, K, T2> *>(&A);+ mma(D16[0], A16[0], B);+ mma(D16[1], A16[1], B);+ }++ static __device__ __forceinline__ void mma(+ tile<32, 8, float> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) {+#if defined(VOLTA_MMA_AVAILABLE)+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ int * Dxi = (int *) D.x;+ asm("mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 "+ "{%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9}, {%10, %11}, {%0, %1, %2, %3, %4, %5, %6, %7};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]), "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]), "r"(Bxi[1]));+ asm("mma.sync.aligned.m8n8k4.row.col.f32.f16.f16.f32 "+ "{%0, %1, %2, %3, %4, %5, %6, %7}, {%8, %9}, {%10, %11}, {%0, %1, %2, %3, %4, %5, %6, %7};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3]), "+r"(Dxi[4]), "+r"(Dxi[5]), "+r"(Dxi[6]), "+r"(Dxi[7])+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]), "r"(Bxi[3]));+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // defined(VOLTA_MMA_AVAILABLE)+ }++ static __device__ __forceinline__ void mma(+ tile<32, 4, half2> & D, const tile<32, 4, half2> & A, const tile<8, 4, half2, DATA_LAYOUT_J_MAJOR_MIRRORED> & B) {+#if defined(VOLTA_MMA_AVAILABLE)+ const int * Axi = (const int *) A.x;+ const int * Bxi = (const int *) B.x;+ int * Dxi = (int *) D.x;+ asm("mma.sync.aligned.m8n8k4.row.row.f16.f16.f16.f16 "+ "{%0, %1, %2, %3}, {%4, %5}, {%6, %7}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[0]), "r"(Axi[1]), "r"(Bxi[0]), "r"(Bxi[1]));+ asm("mma.sync.aligned.m8n8k4.row.row.f16.f16.f16.f16 "+ "{%0, %1, %2, %3}, {%4, %5}, {%6, %7}, {%0, %1, %2, %3};"+ : "+r"(Dxi[0]), "+r"(Dxi[1]), "+r"(Dxi[2]), "+r"(Dxi[3])+ : "r"(Axi[2]), "r"(Axi[3]), "r"(Bxi[2]), "r"(Bxi[3]));+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // defined(VOLTA_MMA_AVAILABLE)+ }++ static __device__ __forceinline__ void mma(+ tile<16, 16, half2, DATA_LAYOUT_I_MAJOR> & D, const tile<32, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & A,+ const tile<16, 8, half2, DATA_LAYOUT_I_MAJOR_MIRRORED> & B) {+#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)+ using halfx16_t = __attribute__((ext_vector_type(16))) _Float16;+ halfx16_t * xD = (halfx16_t *) D.x;+ const halfx16_t * xA = (const halfx16_t *) A.x;+ const halfx16_t * xB = (const halfx16_t *) B.x;+ xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[0], xB[0], xD[0], /*opsel =*/ 0);+ xD[0] = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32(xA[1], xB[0], xD[0], /*opsel =*/ 1);+#else+ GGML_UNUSED_VARS(D, A, B);+ NO_DEVICE_CODE;+#endif // TURING_MMA_AVAILABLE+ }++ template <data_layout dl_d, data_layout dl_ab>+ static __device__ __forceinline__ void mma(+ tile<16, 16, int, dl_d> & D, const tile<16, 4, int, dl_ab> & A, const tile<16, 4, int, dl_ab> & B) {+#if defined(AMD_MFMA_AVAILABLE)+ using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;+ int32x4_t * acc = (int32x4_t *) D.x;+#if defined(CDNA4) || defined(CDNA3)+ const int64_t xA = uint32_t(A.x[0]);+ const int64_t xB = uint32_t(B.x[0]);+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x32_i8(xA, xB, acc[0], 0, 0, 0);+#elif defined(CDNA2) || defined(CDNA1)+ acc[0] = __builtin_amdgcn_mfma_i32_16x16x16i8(A.x[0], B.x[0], acc[0], 0, 0, 0);+#endif // defined(CDNA4) || defined(CDNA3)+#elif defined(AMD_WMMA_AVAILABLE)+ using int32x8_t = __attribute__((__vector_size__(8 * sizeof(int)))) int;+ int32x8_t * acc = (int32x8_t *) D.x;+#if defined(RDNA4)+ using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int;+ int32x2_t * a_vec = (int32x2_t *) A.x;+ int32x2_t * b_vec = (int32x2_t *) B.x;+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12(true, a_vec[0], true, b_vec[0], acc[0], false);+#elif defined(RDNA3)+ using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int;+ int32x4_t * a_vec = (int32x4_t *) A.x;+ int32x4_t * b_vec = (int32x4_t *) B.x;+ acc[0] = __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32(true, a_vec[0], true, b_vec[0], acc[0], false);+#endif // RDNA4+#else+ GGML_UNUSED(D);+ GGML_UNUSED(A);+ GGML_UNUSED(B);+ NO_DEVICE_CODE;+#endif // AMD_WMMA_AVAILABLE+ }+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cunew file mode 100644index 00000000..345b39eb--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cu@@ -0,0 +1,221 @@+#include "common.cuh"+#include "mmid.cuh"++// To reduce shared memory use, store "it" and "iex_used" with 22/10 bits each.+struct mm_ids_helper_store {+ uint32_t data;++ __device__ mm_ids_helper_store(const uint32_t it, const uint32_t iex_used) {+ data = (it & 0x003FFFFF) | (iex_used << 22);+ }++ __device__ uint32_t it() const {+ return data & 0x003FFFFF;+ }++ __device__ uint32_t iex_used() const {+ return data >> 22;+ }+};+static_assert(sizeof(mm_ids_helper_store) == 4, "unexpected size for mm_ids_helper_store");++// the generic path passes 0, which needs no padding since it never groups lanes by token+template <int n> struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2<(n + 1)/2>::value; };+template <> struct mm_ids_pow2<1> { static constexpr int value = 1; };+template <> struct mm_ids_pow2<0> { static constexpr int value = 1; };++// Helper function for mul_mat_id, converts ids to a more convenient format.+// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert.+// ids_dst describes the same mapping but for the dst tensor.+// The upper and lower bounds for the ith expert in the compact src1 tensor are stored in expert_bounds[i:i+1].+template <int n_expert_used_template>+__launch_bounds__(ggml_cuda_get_physical_warp_size(), 1)+static __global__ void mm_ids_helper(+ const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,+ const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template;+ const int expert = blockIdx.x;++ extern __shared__ char data_mm_ids_helper[];+ mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper;++ int nex_prev = 0; // Number of columns for experts with a lower index.+ int it_compact = 0; // Running index for the compact slice of this expert.++ if constexpr (n_expert_used_template == 0) {+ // Generic implementation:+ for (int it = 0; it < n_tokens; ++it) {+ int iex_used = -1; // The index at which the expert is used, if any.+ for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) {+ const int expert_used = ids[it*si1 + iex];+ nex_prev += expert_used < expert;+ if (expert_used == expert) {+ iex_used = iex;+ }+ }++ if (iex_used != -1) {+ store[it_compact] = mm_ids_helper_store(it, iex_used);+ }++ if (warp_reduce_any<warp_size>(iex_used != -1)) {+ it_compact++;+ }+ }+ } else {+ // Implementation optimized for specific numbers of experts used:+ // a warp holds a whole number of token slots, so the slot count is padded to a power of 2+ constexpr int neu_padded = mm_ids_pow2<n_expert_used_template>::value;+ static_assert(neu_padded <= warp_size && warp_size % neu_padded == 0, "bad n_expert_used");+ for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) {+ const int it = it0 + threadIdx.x / neu_padded;++ const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.+ const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?+ ids[it*si1 + iex] : INT_MAX;+ const int iex_used = expert_used == expert ? iex : -1;+ nex_prev += expert_used < expert;++ // Whether the threads at this token position have used the expert:+ const int it_compact_add_self = warp_reduce_any<neu_padded>(iex_used != -1);++ // Do a scan over threads at lower token positions in warp to get the correct index for writing data:+ int it_compact_add_lower = 0;+#pragma unroll+ for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {+ const int tmp = __shfl_up_sync(0xFFFFFFFF, it_compact_add_self, offset, warp_size);+ if (threadIdx.x >= static_cast<unsigned int>(offset)) {+ it_compact_add_lower += tmp;+ }+ }++ if (iex_used != -1) {+ store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used);+ }++ // The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:+ it_compact += __shfl_sync(0xFFFFFFFF, it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);+ }+ }+ nex_prev = warp_reduce_sum<warp_size>(nex_prev);+ ggml_cuda_syncwarp();++ for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {+ const mm_ids_helper_store store_it = store[itc];+ const int it = store_it.it();+ const int iex_used = store_it.iex_used();+ ids_dst[nex_prev + itc] = it*n_expert_used + iex_used;+ // ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup+ if (write_inverse) {+ ids_src1[it*n_expert_used + iex_used] = nex_prev + itc;+ } else {+ ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y;+ }+ }++ if (threadIdx.x != 0) {+ return;+ }++ expert_bounds[expert] = nex_prev;++ if (expert < static_cast<int>(gridDim.x) - 1) {+ return;+ }++ expert_bounds[gridDim.x] = nex_prev + it_compact;+}++template <int n_expert_used_template>+static void launch_mm_ids_helper(+ const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,+ const int n_experts, const int n_tokens, const int n_expert_used_var, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) {+ GGML_ASSERT(n_tokens < (1 << 22) && "too few bits in mm_ids_helper_store");+ GGML_ASSERT(n_expert_used_var < (1 << 10) && "too few bits in mm_ids_helper_store");++ const int id = ggml_cuda_get_device();+ const int warp_size = ggml_cuda_info().devices[id].warp_size;+ const size_t smpbo = ggml_cuda_info().devices[id].smpbo;+ CUDA_SET_SHARED_MEMORY_LIMIT(mm_ids_helper<n_expert_used_template>, smpbo);++ const dim3 num_blocks(n_experts, 1, 1);+ const dim3 block_size(warp_size, 1, 1);+ const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store);+ GGML_ASSERT(nbytes_shared <= smpbo);+ mm_ids_helper<n_expert_used_template><<<num_blocks, block_size, nbytes_shared, stream>>>+ (ids, ids_src1, ids_dst, expert_bounds, n_tokens, n_expert_used_var, nchannels_y, si1, sis1, write_inverse);+}++void ggml_cuda_launch_mm_ids_helper(+ const int32_t * __restrict__ ids, int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, int32_t * __restrict__ expert_bounds,+ const int n_experts, const int n_tokens, const int n_expert_used, const int nchannels_y, const int si1, const int sis1, const bool write_inverse, cudaStream_t stream) {+ switch (n_expert_used) {+ case 2:+ launch_mm_ids_helper< 2>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);+ break;+ case 4:+ launch_mm_ids_helper< 4>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);+ break;+ case 6:+ launch_mm_ids_helper< 6>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);+ break;+ case 8:+ launch_mm_ids_helper< 8>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);+ break;+ case 10:+ launch_mm_ids_helper<10>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);+ break;+ case 16:+ launch_mm_ids_helper<16>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);+ break;+ case 32:+ launch_mm_ids_helper<32>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);+ break;+ default:+ launch_mm_ids_helper< 0>(ids, ids_src1, ids_dst, expert_bounds, n_experts, n_tokens, n_expert_used, nchannels_y, si1, sis1, write_inverse, stream);+ break;+ }+}++// Lists the (expert, column tile) pairs that hold columns for a flat MoE grid, in expert order.+// The other ntiles grid columns get tile jt_none of expert 0, which starts past its columns, so mul_mat_q skips them.+template <int nthreads>+__launch_bounds__(nthreads, 1)+static __global__ void mm_ids_tile_map(+ const int32_t * __restrict__ expert_bounds, int2 * __restrict__ tile_map, const int n_experts, const int J, const int ntiles, const int jt_none) {+ __shared__ int scan[nthreads];++ int start = 0; // First tile of the current chunk of experts.+ for (int e0 = 0; e0 < n_experts; e0 += nthreads) {+ const int e = e0 + threadIdx.x;+ const int n = e < n_experts ? (expert_bounds[e + 1] - expert_bounds[e] + J - 1) / J : 0;++ scan[threadIdx.x] = n;+ __syncthreads();+#pragma unroll+ for (int offset = 1; offset < nthreads; offset *= 2) {+ const int add = threadIdx.x >= offset ? scan[threadIdx.x - offset] : 0;+ __syncthreads();+ scan[threadIdx.x] += add;+ __syncthreads();+ }++ const int first = start + scan[threadIdx.x] - n;+ for (int jt = 0; jt < n; ++jt) {+ tile_map[first + jt] = make_int2(e, jt);+ }+ start += scan[nthreads - 1];+ __syncthreads();+ }++ for (int t = start + threadIdx.x; t < ntiles; t += nthreads) {+ tile_map[t] = make_int2(0, jt_none);+ }+}++void ggml_cuda_launch_mm_ids_tile_map(+ const int32_t * expert_bounds, int2 * tile_map, const int n_experts, const int J, const int ntiles, const int jt_none, cudaStream_t stream) {+ constexpr int nthreads = 512;+ mm_ids_tile_map<nthreads><<<1, nthreads, 0, stream>>>(expert_bounds, tile_map, n_experts, J, ntiles, jt_none);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cuhnew file mode 100644index 00000000..b0e8edb8--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmid.cuh@@ -0,0 +1,8 @@+#pragma once++void ggml_cuda_launch_mm_ids_helper(+ const int32_t * ids, int32_t * ids_src1, int32_t * ids_dst, int32_t * expert_bounds,+ int n_experts, int n_tokens, int n_expert_used, int nchannels_y, int si1, int sis1, bool write_inverse, cudaStream_t stream);++void ggml_cuda_launch_mm_ids_tile_map(+ const int32_t * expert_bounds, int2 * tile_map, int n_experts, int J, int ntiles, int jt_none, cudaStream_t stream);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-ampere.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-ampere.cuhnew file mode 100644index 00000000..9f9fd197--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-ampere.cuh@@ -0,0 +1,383 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_ampere(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-blackwell.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-blackwell.cuhnew file mode 100644index 00000000..9fbe32b6--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-blackwell.cuh@@ -0,0 +1,37 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_blackwell(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_MXFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);++ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, true);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);+ CASE(GGML_TYPE_NVFP4, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, MMQ_ITER_K_FP4, true, false);++ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-cdna.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-cdna.cuhnew file mode 100644index 00000000..4a8d89f7--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-cdna.cuh@@ -0,0 +1,185 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_cdna(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_1, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q8_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q2_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q3_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q4_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q5_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_Q6_K, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ2_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ3_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_IQ4_NL, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_MXFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);++ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);+ CASE(GGML_TYPE_NVFP4, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-gcn.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-gcn.cuhnew file mode 100644index 00000000..24af2ef2--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-gcn.cuh@@ -0,0 +1,281 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-dp4a.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-dp4a.cuhnew file mode 100644index 00000000..83eb7c14--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-dp4a.cuh@@ -0,0 +1,273 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_pascal_dp4a(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-older.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-older.cuhnew file mode 100644index 00000000..2a8dc9e1--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-pascal-older.cuh@@ -0,0 +1,273 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_pascal_older(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna2.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna2.cuhnew file mode 100644index 00000000..8324d9e1--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna2.cuh@@ -0,0 +1,273 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna2(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3-5.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3-5.cuhnew file mode 100644index 00000000..ead3825d--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3-5.cuh@@ -0,0 +1,290 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3_5(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3.cuhnew file mode 100644index 00000000..3a3ef7bd--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna3.cuh@@ -0,0 +1,274 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 4, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 128, 4, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 128, 4, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 4, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 4, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 4, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 128, 4, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 1, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 128, 1, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 1, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 1, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna4.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna4.cuhnew file mode 100644index 00000000..9293d9d5--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-config-rdna4.cuh@@ -0,0 +1,290 @@+static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna4(ggml_type type, int J, bool fallback) {+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);++// ---------------------------------------------------------------------------------------------++ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);++ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);+ CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);++ return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuhnew file mode 100644index 00000000..017b93b5--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-load-tiles.cuh@@ -0,0 +1,1768 @@+#pragma once++#include "vecdotq.cuh"++#include "mmq.cuh"++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q1_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int blocks_per_iter = MMQ_ITER_K / QK1_0;+ constexpr int threads_per_row = blocks_per_iter * QI1_0;+ constexpr int nrows = warp_size / threads_per_row;+ constexpr int scale_entries_per_block = QK1_0 / QK8_1;+ constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block;++ const int txi = threadIdx.x % threads_per_row;+ const int kbx = txi / QI1_0;+ const int kqsx = txi % QI1_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + kbx;+ const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 2;++ const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0;+#pragma unroll+ for (int j = 0; j < 2; ++j) {+ const int q = qxi[j];++ // unpack crumbs into nibble indices+ const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13]+ const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15]+ // unpack nibbles into byte values+ const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0);+ const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0);+ const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16);+ const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16);+ // unshuffle values+ const int v0 = __byte_perm(s0, s1, 0x5410);+ const int v1 = __byte_perm(s0, s1, 0x7632);+ const int v2 = __byte_perm(s2, s3, 0x5410);+ const int v3 = __byte_perm(s2, s3, 0x7632);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + dst_offset + j*4+0] = v0;+ x_qs[i*sram_stride + dst_offset + j*4+1] = v1;+ x_qs[i*sram_stride + dst_offset + j*4+2] = v2;+ x_qs[i*sram_stride + dst_offset + j*4+3] = v3;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+0] = v0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+1] = v1;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+2] = v2;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*4+3] = v3;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+ }++ const int ksx = threadIdx.x % scale_entries_per_row;+ const int scale_block = ksx / scale_entries_per_block;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps) {+ int i = i0 + threadIdx.y;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q1_0 * bxi = (const block_q1_0 *) x + kbx0 + i*stride + scale_block;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + ksx] = bxi->d;+#else+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int blocks_per_iter = MMQ_ITER_K / QK2_0;+ constexpr int threads_per_row = blocks_per_iter * QI2_0;+ constexpr int nrows = warp_size / threads_per_row;+ constexpr int scale_entries_per_block = QK2_0 / QK8_1;+ constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block;++ const int txi = threadIdx.x % threads_per_row;+ const int kbx = txi / QI2_0;+ const int kqsx = txi % QI2_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx;+ const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 4;++ const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0;++#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int q = qxi[j];++#if defined(GGML_USE_HIP)+ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);+ const uint32_t qy_bits = q >> 8;+ const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);+ const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);+ const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);+#else+ // unpack even and odd crumbs into byte values+ const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);+ const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);+ // unshuffle values+ const int qx = __byte_perm(qe, qo, 0x5140);+ const int qy = __byte_perm(qe, qo, 0x7362);+#endif // defined(GGML_USE_HIP)++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + dst_offset + j*2+0] = qx;+ x_qs[i*sram_stride + dst_offset + j*2+1] = qy;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+0] = qx;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+1] = qy;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+ }++ const int ksx = threadIdx.x % scale_entries_per_row;+ const int scale_block = ksx / scale_entries_per_block;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps) {+ int i = i0 + threadIdx.y;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + scale_block;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + ksx] = bxi->d;+#else+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_0);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI4_0;+ const int kqsx = txi % QI4_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbx;+ const int qs0 = get_int_b2(bxi->qs, kqsx);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + 0] = __vsub4((qs0 >> 0) & 0x0F0F0F0F, 0x08080808);+ x_qs[i*sram_stride + kbx*(2*QI4_0) + kqsx + QI4_0] = __vsub4((qs0 >> 4) & 0x0F0F0F0F, 0x08080808);+#else+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_0;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_0 * bxi = (const block_q4_0 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = bxi->d;+#else+ x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + kbxd] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_1(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_1);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI4_1;+ const int kqsx = txi % QI4_1;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbx;+ const int qs0 = get_int_b4(bxi->qs, kqsx);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + 0] = (qs0 >> 0) & 0x0F0F0F0F;+ x_qs[i*sram_stride + kbx*(2*QI4_1) + kqsx + QI4_1] = (qs0 >> 4) & 0x0F0F0F0F;+#else+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_1;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_1 * bxi = (const block_q4_1 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_dm[i*sram_stride + kbxd] = bxi->dm;+#else+ x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + kbxd] = bxi->dm;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_0);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI5_0;+ const int kqsx = txi % QI5_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbx;++ const int ql = get_int_b2(bxi->qs, kqsx);+ const int qh = get_int_b2(bxi->qh, 0) >> (4 * kqsx);++ int qs0 = (ql >> 0) & 0x0F0F0F0F;+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28+ qs0 = __vsub4(qs0, 0x10101010); // subtract 16++ int qs1 = (ql >> 4) & 0x0F0F0F0F;+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28+ qs1 = __vsub4(qs1, 0x10101010); // subtract 16++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + 0] = qs0;+ x_qs[i*sram_stride + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + 0] = qs0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_0) + kqsx + QI5_0] = qs1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_0;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_0 * bxi = (const block_q5_0 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = bxi->d;+#else+ x_df[i*(MMQ_TILE_NE_K/QI5_0) + i/QI5_0 + kbxd] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_1(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_1);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI5_1;+ const int kqsx = txi % QI5_1;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbx;++ const int ql = get_int_b4(bxi->qs, kqsx);+ const int qh = get_int_b4(bxi->qh, 0) >> (4 * kqsx);++ int qs0 = (ql >> 0) & 0x0F0F0F0F;+ qs0 |= (qh << 4) & 0x00000010; // 0 -> 4+ qs0 |= (qh << 11) & 0x00001000; // 1 -> 12+ qs0 |= (qh << 18) & 0x00100000; // 2 -> 20+ qs0 |= (qh << 25) & 0x10000000; // 3 -> 28++ int qs1 = (ql >> 4) & 0x0F0F0F0F;+ qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4+ qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12+ qs1 |= (qh << 2) & 0x00100000; // 18 -> 20+ qs1 |= (qh << 9) & 0x10000000; // 19 -> 28++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + 0] = qs0;+ x_qs[i*sram_stride + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + 0] = qs0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kbx*(2*QI5_1) + kqsx + QI5_1] = qs1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI5_1;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_1 * bxi = (const block_q5_1 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_dm[i*sram_stride + kbxd] = bxi->dm;+#else+ x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + kbxd] = bxi->dm;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q8_0(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_tile + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ // MMQ_ITER_K / (4 * QR8_0) == 64 required. but NV has only 32 threads per warp+ constexpr int threads_per_row = 32;+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI8_0;+ const int kqsx = txi % QI8_0;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbx;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);+ x_qs[i*sram_stride + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 0 + txi] = get_int_b2(bxi[0].qs, kqsx);+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + MMQ_TILE_NE_K + txi] = get_int_b2(bxi[MMQ_TILE_NE_K/QI8_0].qs, kqsx);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = 2*MMQ_TILE_NE_K / QI8_0;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q8_0 * bxi = (const block_q8_0 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = bxi->d;+#else+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + kbxd] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++// ---------------------------------------------------------------------------------------------++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR2_K);+ constexpr int nrows = ggml_cuda_get_physical_warp_size() / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q2_K * bxi = (const block_q2_K *) x + kbx0 + i*stride;++ const int x_ql_0 = get_int_b2(bxi->qs, kqsx);++#pragma unroll+ for (int l = 0; l < QR2_K; ++l) {+ const int k = (kqsx/8)*32 + l*8 + kqsx % 8;++ const int x_qs_k = (x_ql_0 >> (2*l)) & 0x03030303;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k] = x_qs_k;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int sc_m = bxi->scales[kqsx];+#ifdef FAST_FP16_AVAILABLE+ const half2 x_dm_ik = __hmul2(bxi->dm, make_half2(sc_m & 0x0F, sc_m >> 4));+#else+ const float2 bxi_dmf = __half22float2(bxi->dm);+ const half2 x_dm_ik = make_half2(bxi_dmf.x*(sc_m & 0x0F), bxi_dmf.y*(sc_m >> 4));+#endif // FAST_FP16_AVAILABLE++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_dm[i*sram_stride + kqsx] = x_dm_ik;+#else+ x_dm[i*(MMQ_TILE_NE_K + 1) + kqsx] = x_dm_ik;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+ int * x_sc = (int *) (x_df + txs.dm);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR3_K);+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;++ const int x_ql_0 = get_int_b2(bxi->qs, kqsx);+ const int x_qh_0 = get_int_b2(bxi->hmask, kqsx % (QI3_K/2)) >> (4 * (kqsx / (QI3_K/2)));++#pragma unroll+ for (int l = 0; l < QR3_K; ++l) {+ const int k = (kqsx/8)*32 + l*8 + kqsx % 8;++ const int x_ql_k = (x_ql_0 >> (2*l)) & 0x03030303;+ const int x_qh_k = ((x_qh_0 >> l) << 2) & 0x04040404;++ const int x_qs_k = __vsub4(x_ql_k | x_qh_k, 0x04040404);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k] = x_qs_k;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k] = x_qs_k;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+ }++ constexpr int rows_per_warp = warp_size / 4;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/4;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;++ const int ksc = threadIdx.x % 4;++ const int ksc_low = ksc % (QI3_K/8);+ const int shift_low = 4 * (ksc / (QI3_K/8));+ const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;++ const int ksc_high = QI3_K/8;+ const int shift_high = 2 * ksc;+ const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;++ const int sc = __vsub4(sc_low | sc_high, 0x20202020);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ const int8_t * sc8 = (const int8_t *) ≻+ const float d = bxi->d;++#pragma unroll+ for (int l = 0; l < int(sizeof(int)); ++l) {+ x_df[i*sram_stride + sizeof(int)*ksc + l] = d*sc8[l];+ }+#else+ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = sc;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#if !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE))+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q3_K * bxi = (const block_q3_K *) x + kbx0 + i*stride;++ x_df[i] = bxi->d;+ }+#endif // !(defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)) || defined(AMD_WMMA_AVAILABLE)+}++static __device__ __forceinline__ int unpack_scales_q45_K(const int * scales, const int ksc) {+ // scale arrangement after the following two lines:+ // - ksc == 0: sc0, sc1, sc2, sc3+ // - ksc == 1: sc4, sc5, sc6, sc7+ // - ksc == 2: m0, m1, m2, m3+ // - ksc == 3: m4, m5, m6, m7+ return ((scales[(ksc%2) + (ksc!=0)] >> (4 * (ksc & (ksc/2)))) & 0x0F0F0F0F) | // lower 4 bits+ ((scales[ksc/2] >> (2 * (ksc % 2))) & 0x30303030); // upper 2 bits+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + 2*MMQ_TILE_NE_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+ int * x_sc = (int *) (x_dm + txs.dm);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_K);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;+ const int qs0 = get_int_b4(bxi->qs, txi);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 0] = (qs0 >> 0) & 0x0F0F0F0F;+ x_qs[i*sram_stride + 16*(txi/8) + txi % 8 + 8] = (qs0 >> 4) & 0x0F0F0F0F;+#else+ x_qs[i*(MMQ_TILE_NE_K + 1) + txi] = qs0;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int rows_per_warp = warp_size / 2;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ // Need if on AMD instead of % because warp_size == 64+ // This causes double work and throughput loss (MI300X)+ // H100 loses about 100 t/s with 'if' condition over '%'+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;+ if (i < I) {+#else+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I;+ {+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;++ const int * scales = (const int *) bxi->scales;+ const int ksc = threadIdx.x % 2;++ const int sc32 = unpack_scales_q45_K(scales, ksc + 0);+ const int m32 = unpack_scales_q45_K(scales, ksc + 2);++ const uint8_t * sc8 = (const uint8_t *) &sc32;+ const uint8_t * m8 = (const uint8_t *) &m32;++ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);++ #pragma unroll+ for (int l = 0; l < int(sizeof(int)); ++l) {+ x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);+ }+ }+ }+#else+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride;++ x_dm[i] = bxi->dm;+ }+ constexpr int rows_per_warp = warp_size / 4;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q4_K * bxi = (const block_q4_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / (QI4_K/8);++ const int * scales = (const int *) bxi->scales;++ const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);+ const int scales8 = unpack_scales_q45_K(scales, ksc);++ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;+ }+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q5_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I);+ int * x_qs = (int *) x_tile;+ half2 * x_dm = (half2 *) (x_qs + txs.qs);+ int * x_sc = (int *) (x_dm + txs.dm);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR5_K);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;+ const int ky = QR5_K*txi;++ const int ql = get_int_b4(bxi->qs, txi);+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;++ const int qh = get_int_b4(bxi->qh, txi % (QI5_K/4));+ const int qh0 = ((qh >> (2 * (txi / (QI5_K/4)) + 0)) << 4) & 0x10101010;+ const int qh1 = ((qh >> (2 * (txi / (QI5_K/4)) + 1)) << 4) & 0x10101010;++ const int kq0 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + 0;+ const int kq1 = ky - ky % (QI5_K/2) + txi % (QI5_K/4) + QI5_K/4;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kq0] = ql0 | qh0;+ x_qs[i*sram_stride + kq1] = ql1 | qh1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = ql0 | qh0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = ql1 | qh1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int rows_per_warp = warp_size / 2;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+#if defined(AMD_MFMA_AVAILABLE)+ // Need if on AMD instead of % because warp_size == 64+ // This causes double work and throughput loss (MI300X)+ // H100 loses about 100 t/s with 'if' condition over '%'+ int i = i0 + threadIdx.y*rows_per_warp + threadIdx.x/2;+ if (i < I) {+#else+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/2) % I;+ {+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;++ const int * scales = (const int *) bxi->scales;+ const int ksc = threadIdx.x % 2;++ const int sc32 = unpack_scales_q45_K(scales, ksc + 0);+ const int m32 = unpack_scales_q45_K(scales, ksc + 2);++ const uint8_t * sc8 = (const uint8_t *) &sc32;+ const uint8_t * m8 = (const uint8_t *) &m32;++ const half2 dm = bxi->dm * make_half2(1.0f, -1.0f);++#pragma unroll+ for (int l = 0; l < int(sizeof(int)); ++l) {+ x_dm[i*sram_stride + sizeof(int)*ksc + l] = dm*make_half2(sc8[l], m8[l]);+ }+ }+ }+#else+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;++ x_dm[i] = bxi->dm;+ }++ constexpr int rows_per_warp = warp_size / 4;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q5_K * bxi = (const block_q5_K *) x + kbx0 + i*stride;++ const int * scales = (const int *) bxi->scales;++ const int ksc = threadIdx.x % (MMQ_TILE_NE_K/8);+ const int scales8 = unpack_scales_q45_K(scales, ksc);++ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + ksc] = scales8;+ }+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q6_K(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+ int * x_sc = (int *) (x_df + MMQ_TILE_NE_K/QI6_K);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+ int * x_sc = (int *) (x_df + txs.dm);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR6_K);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;++ const int ql = get_int_b2(bxi->ql, txi);+ const int ql0 = (ql >> 0) & 0x0F0F0F0F;+ const int ql1 = (ql >> 4) & 0x0F0F0F0F;++ const int qh = get_int_b2(bxi->qh, (QI6_K/4) * (txi / (QI6_K/2)) + txi % (QI6_K/4));+ const int qh0 = ((qh >> ((txi & 0x08) >> 2)) << 4) & 0x30303030;+ const int qh1 = (qh >> ((txi & 0x08) >> 2)) & 0x30303030;++ const int kq0 = 2*txi - txi % (QI6_K/2) + 0;+ const int kq1 = 2*txi - txi % (QI6_K/2) + QI6_K/2;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kq0] = __vsub4(ql0 | qh0, 0x20202020);+ x_qs[i*sram_stride + kq1] = __vsub4(ql1 | qh1, 0x20202020);+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq0] = __vsub4(ql0 | qh0, 0x20202020);+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + kq1] = __vsub4(ql1 | qh1, 0x20202020);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*warp_size) {+ int i = (i0 + threadIdx.y*warp_size + threadIdx.x) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride] = bxi->d;+#else+ x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K] = bxi->d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int rows_per_warp = warp_size / 4;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps*rows_per_warp) {+ int i = (i0 + threadIdx.y*rows_per_warp + threadIdx.x/(MMQ_TILE_NE_K/8)) % I;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_q6_K * bxi = (const block_q6_K *) x + kbx0 + i*stride + (threadIdx.x % (MMQ_TILE_NE_K/8)) / 4;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_sc[i*sram_stride + threadIdx.x%4] = get_int_b2(bxi->scales, threadIdx.x % (MMQ_TILE_NE_K/8));+#else+ x_sc[i*(MMQ_TILE_NE_K/8) + i/8 + threadIdx.x%(MMQ_TILE_NE_K/8)] = get_int_b2(bxi->scales, threadIdx.x%(QI6_K/8));+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++// ---------------------------------------------------------------------------------------------++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_s(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I);+ int * x_qs = (int *) x_tile;+ half2 * x_ds = (half2 *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_S);+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq1_s * bxi = (const block_iq1_s *) x + kbx0 + i*stride;++ const int qs_packed = get_int_b2(bxi->qs, kqsx);+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bxi->qh[kqsx];++ #pragma unroll+ for (int l = 0; l < QR1_S/2; ++l) {+ const int grid = iq1s_grid_gpu[qs[l] | (((qh >> (3*l)) & 0x07) << 8)];++ const int grid0 = (grid >> 0) & 0x0F0F0F0F;+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid0;+ x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const float d1q = __half2float(bxi->d) * (((qh >> 11) & 0x0E) + 1);+ const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_ds[i*sram_stride + kqsx] = make_half2(d1q, d1q*delta);+#else+ x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XXS, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XXS)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq2_xxs * bxi = (const block_iq2_xxs *) x + kbx0 + i*stride;++ const int q2 = get_int_b2(bxi->qs, 2*kqsx+0);+ const uint8_t * aux8 = (const uint8_t *) &q2;+ const uint32_t aux32 = get_int_b2(bxi->qs, 2*kqsx+1);++#pragma unroll+ for (int l = 0; l < QR2_XXS; ++l) {+ const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[l]];+ const uint32_t signs = unpack_ksigns(aux32 >> (7 * l));++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid0;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid1;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid0;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid1;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = aux32 >> 27 | 1; // (scale * 2 + 1)+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4+#else+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = d * ls / 8; // (d * scale + d / 2) / 4+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_XS, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_XS)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq2_xs * bxi = (const block_iq2_xs *) x + kbx0 + i*stride;++ const int2 q2_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));+ const uint16_t * q2 = (const uint16_t *) &q2_packed;++ #pragma unroll+ for (int l = 0; l < QR2_XS; ++l) {+ const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l] & 0x1FF];+ const uint32_t signs = unpack_ksigns(q2[l] >> 9);++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = bxi->scales[kqsx];+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;+ x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+#else+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_s(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ2_S, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR2_S)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq2_s * bxi = (const block_iq2_s *) x + kbx0 + i*stride;++ const int qs_packed = get_int_b2(bxi->qs, kqsx);+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bxi->qh[kqsx];++ const int signs_packed_32 = get_int_b2(bxi->qs, QK_K/32 + kqsx);+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;++#pragma unroll+ for (int l = 0; l < QR2_S; ++l) {+ const int * grid_pos = (const int *)(iq2s_grid + (qs[l] | ((qh << (8-2*l)) & 0x300)));++ const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);+ const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);++ const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = bxi->scales[kqsx];+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;+ x_df[i*sram_stride + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+#else+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+0] = ((ls & 0x0F)*d + d/2)/4;+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + 2*kqsx+1] = ((ls >> 4)*d + d/2)/4;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_xxs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_XXS, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_XXS)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq3_xxs * bxi = (const block_iq3_xxs *) x + kbx0 + i*stride;++ const int2 q3_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));+ const uint8_t * q3 = (const uint8_t *) &q3_packed;+ const uint32_t aux32 = get_int_b2(bxi->qs, QK_K/16 + kqsx);++#pragma unroll+ for (int l = 0; l < QR3_XXS; ++l) {+ const int2 grid_pos = make_int2(iq3xxs_grid[q3[2*l+0]], iq3xxs_grid[q3[2*l+1]]);+ const uint32_t signs = unpack_ksigns(aux32 >> (7*l));++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*sram_stride + 8*kqsx + (2*l + 1)] = grid_h;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 0)] = grid_l;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l + 1)] = grid_h;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = aux32 >> 28;+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kqsx] = (ls*d + d/2)/2;+#else+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = (ls*d + d/2)/2;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq3_s(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = (MMQ_ITER_K / (4 * QR3_S)) / 2;+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq3_s * bxi = (const block_iq3_s *) x + kbx0 + i*stride;++ const int2 qs_packed = make_int2(get_int_b2(bxi->qs, 2*kqsx+0), get_int_b2(bxi->qs, 2*kqsx+1));+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bxi->qh[kqsx];++ const int signs_packed_32 = get_int_b2(bxi->signs, kqsx);+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;++#pragma unroll+ for (int l = 0; l < QR3_S; ++l) {+ const int2 grid_pos = make_int2(+ iq3s_grid[qs[2*l+0] | ((qh << (8 - 2*l)) & 0x100)],+ iq3s_grid[qs[2*l+1] | ((qh << (7 - 2*l)) & 0x100)]);++ const int signs0 = __vcmpne4(((signs_packed_8[l] & 0x03) << 7) | ((signs_packed_8[l] & 0x0C) << 21), 0x00000000);+ const int signs1 = __vcmpne4(((signs_packed_8[l] & 0x30) << 3) | ((signs_packed_8[l] & 0xC0) << 17), 0x00000000);++ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid_l;+ x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid_h;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid_l;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid_h;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ const int ls = 1 + 2*((bxi->scales[kqsx/2] >> (((2*kqsx) << 1) & 0x04)) & 0x0F);+ const float d = bxi->d;+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kqsx] = ls*d;+#else+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = ls*d;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_xs(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_XS, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_XS);+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;++ const int aux_q4 = get_int_b4(bxi->qs, kqsx);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);+ const int k0 = 8 * (kqsx / 4) + kqsx % 4;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k0 + 0] = v.x;+ x_qs[i*sram_stride + k0 + 4] = v.y;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 4] = v.y;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int rows_per_warp = warp_size / 8;+#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / (MMQ_TILE_NE_K/4);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq4_xs * bxi = (const block_iq4_xs *) x + kbx0 + i*stride;++ const float d = __half2float(bxi->d);++ const int ls = ((bxi->scales_l[(threadIdx.x % 8)/2] >> (4*(threadIdx.x % 2))) & 0x0F)+ | (((bxi->scales_h >> (2*(threadIdx.x % 8))) & 0x03) << 4);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + threadIdx.x % 8] = d * (ls - 32);+#else+ x_df[i*(MMQ_TILE_NE_K/4) + i/4 + threadIdx.x % 8] = d * (ls - 32);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq4_nl(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ4_NL, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI4_NL;+ const int kqsx = txi % QI4_NL;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbx;++ const int aux_q4 = get_int_b2(bxi->qs, kqsx);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);+ const int k0 = kbx * (2 * QI4_NL) + kqsx;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k0 + 0] = v.x;+ x_qs[i*sram_stride + k0 + QI4_NL] = v.y;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_iq4_nl * bxi = (const block_iq4_nl *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = __half2float(bxi->d);+#else+ x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++// ---------------------------------------------------------------------------------------------++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_MXFP4, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / (4 * QR_MXFP4);+ constexpr int nrows = warp_size / threads_per_row;+ const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x;+ const int kbx = txi / QI_MXFP4;+ const int kqsx = txi % QI_MXFP4;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nrows*nwarps) {+ int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row);++ if (fallback) {+ i = min(i, i_max);+ }++ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbx;++ const int aux_q4 = get_int_b1(bxi->qs, kqsx);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);+ const int k0 = kbx * (2 * QI_MXFP4) + kqsx;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + k0 + 0] = v.x;+ x_qs[i*sram_stride + k0 + QI_MXFP4] = v.y;+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x;+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI_MXFP4] = v.y;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI_MXFP4;+ constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row;+ const int kbxd = threadIdx.x % blocks_per_tile_x_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * rows_per_warp) {+ int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i*stride + kbxd;++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;+#else+ x_df[i*(MMQ_TILE_NE_K/QI_MXFP4) + i/QI_MXFP4 + kbxd] = ggml_cuda_e8m0_to_fp32(bxi->e)*0.5f;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_mxfp4_fp4(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4);++ int * x_qs = (int *) x_tile;+ uint32_t * x_sc = (uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);++ const int txi = threadIdx.x;++ constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback, GGML_PREC_Q4);++ constexpr int threads_per_row = iter_k / QK_MXFP4; // each thread processes 1 block+ constexpr int rows_per_warp = warp_size / threads_per_row;+ const int kbx = txi % threads_per_row;+ const int row_in_warp = txi / threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;++ if constexpr (fallback) {+ i = min(i, i_max);+ }++ const block_mxfp4 * bxi = (const block_mxfp4 *) x + kbx0 + i * stride + kbx;++ // quantize_mxfp4_mmq permutes nibbles to match the quantized format+ const int k0 = kbx * 4;+ memcpy(x_qs + i*sram_stride + k0, bxi->qs, 16);++ // Load E8M0 scales: pack 2 consecutive scales into one uint32+ if (kbx % 2 == 0) {+ uint32_t e = bxi->e;+ e |= ((bxi + 1)->e << 8);+ x_sc[i*sram_stride + kbx / 2] = e;+ }+ }+}++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kb0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_NVFP4, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int threads_per_row = MMQ_ITER_K / QK_NVFP4;+ constexpr int rows_per_warp = warp_size / threads_per_row;+ const int kbx = threadIdx.x % threads_per_row;+ const int row_in_warp = threadIdx.x / threads_per_row;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;++ if constexpr (fallback) {+ i = min(i, i_max);+ }++ const block_nvfp4 * bxi = (const block_nvfp4 *) x + kb0 + i * stride + kbx;+ const uint32_t * __restrict__ src_qs = reinterpret_cast<const uint32_t *>(bxi->qs);+ const int kqs = 16 * kbx;+ const int ksc = 4 * kbx;++#pragma unroll+ for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {+ const int2 q0 = get_int_from_table_16(src_qs[2 * sub + 0], kvalues_mxfp4);+ const int2 q1 = get_int_from_table_16(src_qs[2 * sub + 1], kvalues_mxfp4);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + kqs + 4 * sub + 0] = q0.x;+ x_qs[i*sram_stride + kqs + 4 * sub + 1] = q1.x;+ x_qs[i*sram_stride + kqs + 4 * sub + 2] = q0.y;+ x_qs[i*sram_stride + kqs + 4 * sub + 3] = q1.y;+ x_df[i*sram_stride + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]);+#else+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 0] = q0.x;+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 1] = q1.x;+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 2] = q0.y;+ x_qs[i * (2 * MMQ_TILE_NE_K + 1) + kqs + 4 * sub + 3] = q1.y;+ x_df[i * (2 * MMQ_TILE_NE_K * 2 / QI_NVFP4) + i / (QK_NVFP4_SUB / QI_NVFP4) + ksc + sub] = ggml_cuda_ue4m3_to_fp32(bxi->d[sub]);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_nvfp4_nvfp4(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q4) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q4);+ constexpr int iter_k = ggml_cuda_mmq_get_K_vram(type, J, fallback, GGML_PREC_Q4);+ constexpr int threads_per_row = iter_k / QK_NVFP4; // each thread processes 1 block+ constexpr int rows_per_warp = warp_size / threads_per_row;+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4);++ uint32_t * x_u32 = (uint32_t *) x_tile;++ const int txi = threadIdx.x;+ const int kbx = txi % threads_per_row;+ const int row_in_warp = txi / threads_per_row;++ const block_nvfp4 * bxi_base = (const block_nvfp4 *) x + kbx0 + kbx;+ uint32_t * x_u32_scale = x_u32 + 64 + kbx;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += rows_per_warp * nwarps) {+ int i = i0 + threadIdx.y * rows_per_warp + row_in_warp;++ if constexpr (fallback) {+ i = min(i, i_max);+ }++ const block_nvfp4 * bxi = bxi_base + i * stride;++ const uint32_t * src_qs = reinterpret_cast<const uint32_t *>(bxi->qs);++#pragma unroll+ for (int sub = 0; sub < QK_NVFP4 / QK_NVFP4_SUB; ++sub) {+ x_u32[i*sram_stride + 8*kbx + 2 * sub + 0] = src_qs[2 * sub + 0];+ x_u32[i*sram_stride + 8*kbx + 2 * sub + 1] = src_qs[2 * sub + 1];+ }++ x_u32_scale[i*sram_stride] = get_int_b4(bxi->d, 0);+ }+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-vec-dot.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-vec-dot.cuhnew file mode 100644index 00000000..a75dd046--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq-vec-dot.cuh@@ -0,0 +1,1237 @@+#pragma once++#include "vecdotq.cuh"+#include "mma.cuh"++using namespace ggml_cuda_mma;++#include "mmq.cuh"++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_0, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_0*VDR_Q4_0_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;+ const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);++ int u[2*VDR_Q4_0_Q8_1_MMQ];++ constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes();+ constexpr int mcpy_int = max_cpy / sizeof(int);+ static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ");++ int tmp0[4], tmp1[4];++ #pragma unroll+ for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) {+ ggml_cuda_memcpy_1<max_cpy>(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] );+ ggml_cuda_memcpy_1<max_cpy>(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_0 + l0 * mcpy_int]);+ }++ u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3];+ u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3];++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>+ (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_0], u,+ x_df[i*(MMQ_TILE_NE_K/QI4_0) + i/QI4_0 + k0/(QR4_0*QI4_0)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_1, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_1*VDR_Q4_1_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;+ const int kyqs = QI8_1 * ((k01/2) / (QI8_1/2)) + (k01/2) % (QI8_1/2);++ int u[2*VDR_Q4_1_Q8_1_MMQ];++ constexpr int max_cpy = ggml_cuda_get_max_cpy_bytes();+ constexpr int mcpy_int = max_cpy / sizeof(int);+ static_assert(VDR_Q4_0_Q8_1_MMQ == 4, "bad VDR_Q4_0_Q8_1_MMQ");++ int tmp0[4], tmp1[4];++ #pragma unroll+ for (int l0 = 0; l0 < 4 / mcpy_int; ++l0) {+ ggml_cuda_memcpy_1<max_cpy>(tmp0 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + l0 * mcpy_int] );+ ggml_cuda_memcpy_1<max_cpy>(tmp1 + l0 * mcpy_int, &y_qs[j*MMQ_TILE_Y_K + kyqs + QI4_1 + l0 * mcpy_int]);+ }++ u[0]=tmp0[0]; u[2]=tmp0[1]; u[4]=tmp0[2]; u[6]=tmp0[3];+ u[1]=tmp1[0]; u[3]=tmp1[1]; u[5]=tmp1[2]; u[7]=tmp1[3];++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>+ (&x_qs[i*(MMQ_TILE_NE_K + 1) + k0/QR4_1], u,+ x_dm[i*(MMQ_TILE_NE_K/QI4_1) + i/QI4_1 + k0/(QR4_1*QI4_1)], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMQ>+ (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k0 % MMQ_TILE_NE_K],+ x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + k0/QI8_0], y_df[j*MMQ_TILE_Y_K + (k0/QI8_1) % (MMQ_TILE_NE_K/QI8_1)]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback, mmq_q8_1_ds_layout ds_layout>+static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 8, int, input_layout> tile_A;+ typedef tile<16, 8, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;+ const half2 * y_ds = (const half2 *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ float dB;+ const int j = j0 + tile_C::get_j(0);+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {+ dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];+ } else {+ dB = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_A::I + tile_C::get_i(l);+ const float dA = x_df[i*sram_stride + k0/QI8_0];+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA*dB;+ }+ }+ }+ }+#else+ typedef tile<16, 8, int> tile_A;+ typedef tile< 8, 8, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + 2*MMQ_TILE_NE_K;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;+ const half2 * y_ds = (const half2 *) y;++ tile_A A[ntx][MMQ_TILE_NE_K/QI8_0];+ float dA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_0];++ const int i0 = (threadIdx.y/ntx)*rows_per_warp;++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ const int k0 = k00 + k01;++ load_ldmatrix(A[n][k01/QI8_0], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ const int k0 = k00 + k01;++ dA[n][l][k01/QI8_0] = x_df[i*sram_stride + k0/QI8_0];+ }+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ tile_B B;+ float dB[tile_C::ne/2];++ load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {+ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];+ } else {+ dB[l] = __low2float(y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n][k01/QI8_0], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l]*dA[n][l/2][k01/QI8_0]*dB[l%2];+ }+ }+ }+ }+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+}+++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_1, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += VDR_Q8_0_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_1_q8_1_impl<QR5_1*VDR_Q5_1_Q8_1_MMQ>+ (&x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],+ x_dm[i*(MMQ_TILE_NE_K/QI5_1) + i/QI5_1 + k0/QI8_1], y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 8, int, input_layout> tile_A;+ typedef tile<16, 8, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_dm = (const half2 *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ const int j = j0 + tile_C::get_j(0);+ const float2 dsB = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_A::I + tile_C::get_i(l);+ float2 dmA = __half22float2(x_dm[i*sram_stride + k0/QI8_1]);+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.x*dsB.x*C.x[l];+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA.y*dsB.y;+ }+ }+ }+ }+#else+ typedef tile<16, 8, int> tile_A;+ typedef tile< 8, 8, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + 2*MMQ_TILE_NE_K;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_dm = (const half2 *) y;++ tile_A A[ntx][MMQ_TILE_NE_K/QI8_1];+ float2 dmA[ntx][tile_C::ne/2][MMQ_TILE_NE_K/QI8_1];++ const int i0 = (threadIdx.y/ntx)*rows_per_warp;++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ const int k0 = k00 + k01;++ load_ldmatrix(A[n][k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_A::I + tile_C::get_i(2*l);++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ const int k0 = k00 + k01;++ dmA[n][l][k01/QI8_1] = __half22float2(x_dm[i*sram_stride + k0/QI8_1]);+ }+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ tile_B B;+ float2 dsB[tile_C::ne/2];++ load_generic(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K); // faster than load_ldmatrix++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ dsB[l] = __half22float2(y_dm[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n][k01/QI8_1], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].x*dsB[l%2].x*C.x[l];+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dmA[n][l/2][k01/QI8_1].y*dsB[l%2].y;+ }+ }+ }+ }+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+}++// Used for NVFP4, Q3_K, IQ2_S, and IQ2_XS+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_0) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q8_0_16_q8_1_impl<QI8_0>(+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0],+ &y_qs[j*MMQ_TILE_Y_K + k01],+ &x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + k0/(QI8_0/2)],+ y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++// Used for Q3_K, IQ2_S, and IQ2_XS:+template <ggml_type type, int J, bool fallback, ggml_prec prec_src1> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 4, int, input_layout> tile_A;+ typedef tile<16, 4, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ const int j = j0 + tile_C::get_j(0);+ const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * x_df[i*sram_stride + k0/4] * dB;+ }+ }+ }+ }+#elif defined(TURING_MMA_AVAILABLE)++ typedef tile<16, 4, int> tile_A;+ typedef tile<16, 8, int> tile_A_8;+ typedef tile< 8, 4, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, prec_src1);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);++ tile_A A[ntx][8];+ float dA[ntx][tile_C::ne/2][8];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {+ const int k0 = k00 + k01;++ load_ldmatrix(((tile_A_8 *) A[n])[k01/8], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {+ const int k0 = k00 + k01;++ dA[n][l][k01/4] = x_df[i*sram_stride + k0/4];+ }+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {+ tile_B B[2];+ float dB[tile_C::ne/2];++ // Here load_generic is faster than load_ldmatrix.+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C[2];+ mma(C[0], A[n][k01/4 + 0], B[0]);+ mma(C[1], A[n][k01/4 + 1], B[1]);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += dB[l%2]*(C[0].x[l]*dA[n][l/2][k01/4 + 0] + C[1].x[l]*dA[n][l/2][k01/4 + 1]);+ }+ }+ }+ }+#else+ GGML_UNUSED_VARS(x, y, sum, k00);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q2_K, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++ float2 y_df[J/nwarps];+#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ y_df[j0/nwarps] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);+ }++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K/2; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ constexpr int ns = 2;+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq<ns>(+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],+ &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,+ &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);+ }+ }+ }++ // Some compilers fail to unroll the loop over k01 if there is a conditional statement for ns in the inner loop.+ // As a workaround 2 separate loops are used instead.+#pragma unroll+ for (int k01 = MMQ_TILE_NE_K/2; k01 < MMQ_TILE_NE_K; k01 += QR2_K*VDR_Q2_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ constexpr int ns = 1;+ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q2_K_q8_1_impl_mmq<ns>(+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01],+ &x_dm[i*(MMQ_TILE_NE_K + 1) + k0/4], k01 < MMQ_TILE_NE_K/2 ? y_df[j0/nwarps].x : y_df[j0/nwarps].y,+ &y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 4, int, input_layout> tile_A;+ typedef tile<16, 4, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ const int j = j0 + tile_C::get_j(0);+ const float dB = (k01 < MMQ_TILE_NE_K/2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K]).x : __half22float2(y_ds[j*MMQ_TILE_Y_K]).y;+ const float sB = (k01 >= MMQ_TILE_NE_K * 3/4) ? 0+ : (((k01/4)%2) ? __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).y+ : __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]).x);++ tile_C Cm;+ if (k01 >= MMQ_TILE_NE_K * 3/4) {+ tile_A A1;+#pragma unroll+ for (int l = 0; l < tile_A::ne; ++l) {+ A1.x[l] = 0x01010101;+ }+ mma(Cm, A1, B);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C Cd;+ mma(Cd, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);+ const float2 dm = __half22float2(x_dm[i*sram_stride + k0/4]);+ float tmp = Cd.x[l]*dm.x;+ if (k01 >= MMQ_TILE_NE_K * 3/4) {+ tmp -= Cm.x[l]*dm.y;+ }+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*dB;+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= dm.y*sB;+ }+ }+ }+ }+#elif defined(TURING_MMA_AVAILABLE)++ typedef tile<16, 4, int> tile_A;+ typedef tile<16, 8, int> tile_A_8;+ typedef tile< 8, 4, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + MMQ_TILE_NE_K*2;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);++ tile_A A[ntx][8];+ float dA[ntx][tile_C::ne/2][8];+ float mA[ntx][tile_C::ne/2][8];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ const int k0 = k00 + k01;++ load_ldmatrix(((tile_A_8 *) A[n])[k01/QI8_1], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1/2) {+ const int k0 = k00 + k01;++ const float2 dm = __half22float2(x_dm[i*sram_stride + k0/(QI8_1/2)]);++ dA[n][l][k01/(QI8_1/2)] = dm.x;+ mA[n][l][k01/(QI8_1/2)] = dm.y;+ }+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ float2 dB[tile_C::ne/2];++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ dB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K]);+ }++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QI8_1) {+ tile_B B[2];++ // Here load_generic is faster than load_ldmatrix.+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + (k01 + 0), MMQ_TILE_Y_K);+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + (k01 + tile_B::J), MMQ_TILE_Y_K);++ tile_C Cm[2];+ if (k01 >= MMQ_TILE_NE_K * 3/4) {+ tile_A A1;+ A1.x[0] = 0x01010101;+ A1.x[1] = 0x01010101;+ mma(Cm[0], A1, B[0]);+ mma(Cm[1], A1, B[1]);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C Cd[2];++ mma(Cd[0], A[n][k01/4 + 0], B[0]);+ mma(Cd[1], A[n][k01/4 + 1], B[1]);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ float tmp = Cd[0].x[l]*dA[n][l/2][k01/4 + 0] + Cd[1].x[l]*dA[n][l/2][k01/4 + 1];+ if (k01 >= MMQ_TILE_NE_K * 3/4) {+ tmp -= Cm[0].x[l]*mA[n][l/2][k01/4 + 0] + Cm[1].x[l]*mA[n][l/2][k01/4 + 1];+ }+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp*(k01 < MMQ_TILE_NE_K/2 ? dB[l%2].x : dB[l%2].y);+ }+ }+ }++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K * 3/4; k01 += QI8_1) {+ float2 sB[tile_C::ne/2];++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ sB[l] = __half22float2(y_ds[j*MMQ_TILE_Y_K + (1 + k01/QI8_1)]);+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 0]*sB[l%2].x;+ sum[(j0/tile_C::J + n)*tile_C::ne + l] -= mA[n][l/2][k01/4 + 1]*sB[l%2].y;+ }+ }+ }+ }+#else+ GGML_UNUSED_VARS(x, y, sum, k00);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q3_K, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * x_sc = (const int *) x_df + txs.dm;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR3_K*VDR_Q3_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ const int8_t * scales = ((const int8_t *) (x_sc + i*(MMQ_TILE_NE_K/8) + i/8)) + k0/4;++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q3_K_q8_1_impl_mmq(+ &x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], scales,+ x_df[i], y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_K, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * x_sc = (const int *) x_dm + txs.dm;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR4_K*VDR_Q4_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ const uint8_t * sc = (const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/32] + 2*(k01/16);++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q4_K_q8_1_impl_mmq(+ &x_qs[i*(MMQ_TILE_NE_K + 1) + k0/2], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,+ x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q5_K, I);+ const int * x_qs = (const int *) x;+ const half2 * x_dm = (const half2 *) x_qs + txs.qs;+ const int * x_sc = (const int *) x_dm + txs.dm;+ const int * y_qs = (const int *) y + 4;+ const half2 * y_ds = (const half2 *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR5_K*VDR_Q5_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ const uint8_t * sc = ((const uint8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k00/32]) + 2*(k01/16);++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q5_K_q8_1_impl_mmq(+ &x_qs[i*(QR5_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc, sc+8,+ x_dm[i], &y_ds[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8);++ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q6_K, I);+ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + txs.qs;+ const int * x_sc = (const int *) x_df + txs.dm;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++// #pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += QR6_K*VDR_Q6_K_Q8_1_MMQ) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ const int8_t * sc = ((const int8_t *) &x_sc[i * (MMQ_TILE_NE_K/8) + i/8 + k0/16]);++ sum[j0/nwarps*I/warp_size + i0/warp_size] += vec_dot_q6_K_q8_1_impl_mmq(+ &x_qs[i*(QR6_K*MMQ_TILE_NE_K + 1) + k0], &y_qs[j*MMQ_TILE_Y_K + k01], sc,+ x_df[i*(MMQ_TILE_NE_K/QI6_K) + i/QI6_K], &y_df[j*MMQ_TILE_Y_K + k01/QI8_1]);+ }+ }+ }+}++template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ constexpr data_layout input_layout = get_input_data_layout();+ typedef tile<16, 4, int, input_layout> tile_A;+ typedef tile<16, 4, int, input_layout> tile_B;+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;+ const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 4) {+ const int k0 = k00 + k01;++ tile_A A[ntx];+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ load_ldmatrix(A[n], x_qs + (i0 + n*tile_A::I)*sram_stride + k0, sram_stride);+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ tile_B B;+ load_ldmatrix(B, y_qs + j0*MMQ_TILE_Y_K + k01, MMQ_TILE_Y_K);++ const int j = j0 + tile_C::get_j(0);+ const float dB = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C;+ mma(C, A[n], B);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(l);+ const int8_t * sc = (const int8_t *) (x_sc + i*sram_stride + k00/16);+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += C.x[l] * sc[k01/4] * x_df[i*sram_stride] * dB;+ }+ }+ }+ }+#elif defined(TURING_MMA_AVAILABLE)++ typedef tile<16, 4, int> tile_A;+ typedef tile< 8, 4, int> tile_B;+ typedef tile<16, 8, int> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q8);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ y += (threadIdx.y % ntx) * (tile_C::J*MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const float * x_df = (const float *) x_qs + MMQ_TILE_NE_K*2;+ const int * x_sc = (const int *) x_df + MMQ_TILE_NE_K/QI6_K;+ const int * y_qs = (const int *) y + 4;+ const float * y_df = (const float *) y;++ const int i0 = (threadIdx.y / ntx) * (ntx*tile_A::I);++ tile_A A[ntx][8];+ int scA[ntx][tile_C::ne/2][8];+ float dA[ntx][tile_C::ne/2];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {+ const int k0 = k00 + k01;++ load_ldmatrix(A[n][k01/4 + 0], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + 0), sram_stride);+ load_ldmatrix(A[n][k01/4 + 1], x_qs + (i0 + n*tile_A::I)*sram_stride + (k0 + tile_A::J), sram_stride);+ }++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 16) {+ const int k0 = k00 + k01;++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);++ const int sc_packed = x_sc[i*sram_stride + k0/16];+ const int8_t * sc = (const int8_t *) &sc_packed;++#pragma unroll+ for (int ksc = 0; ksc < sizeof(int); ++ksc) {+ scA[n][l][k01/4 + ksc] = sc[ksc];+ }+ }+ }++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int i = i0 + n*tile_C::I + tile_C::get_i(2*l);++ dA[n][l] = x_df[i*sram_stride];+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+ float tmp[ntx][tile_C::ne] = {{0.0f}};++#pragma unroll+ for (int k01 = 0; k01 < MMQ_TILE_NE_K; k01 += 8) {+ tile_B B[2];+ float dB[tile_C::ne/2];++ // Here load_generic is faster than load_ldmatrix.+ load_generic(B[0], y_qs + j0*MMQ_TILE_Y_K + 0 + k01, MMQ_TILE_Y_K);+ load_generic(B[1], y_qs + j0*MMQ_TILE_Y_K + tile_B::J + k01, MMQ_TILE_Y_K);++#pragma unroll+ for (int l = 0; l < tile_C::ne/2; ++l) {+ const int j = j0 + tile_C::get_j(l);++ dB[l] = y_df[j*MMQ_TILE_Y_K + k01/QI8_1];+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ tile_C C[2];+ mma(C[0], A[n][k01/4 + 0], B[0]);+ mma(C[1], A[n][k01/4 + 1], B[1]);++#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ tmp[n][l] += (C[0].x[l]*scA[n][l/2][k01/4 + 0] + C[1].x[l]*scA[n][l/2][k01/4 + 1])*dB[l%2];+ }+ }+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ sum[(j0/tile_C::J + n)*tile_C::ne + l] += tmp[n][l]*dA[n][l/2];+ }+ }+ }+#else+ GGML_UNUSED_VARS(x, y, sum, k00);+ NO_DEVICE_CODE;+#endif // AMD_MFMA_AVAILABLE || AMD_WMMA_AVAILABLE+}++// ---------------------------------------------------------------------------------------------++// Shared MMA kernel for MXFP4 and NVFP4 on Blackwell.+// Both quantizations encode values as e2m1 (FP4) and produce one uint32 scale per+// m16n8k64 MMA call; only the PTX kind (scale_vec::2X ue8m0 vs scale_vec::4X ue4m3)+// and the per-type stride constant differ.+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_vec_dot_fp4_fp4_mma(+ const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00) {++ typedef tile<16, 8, int> tile_A;+ typedef tile<8, 8, int> tile_B;+ typedef tile<16, 8, float> tile_C;++ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q4);+ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, GGML_PREC_Q4);+ constexpr int ntx = rows_per_warp / tile_C::I;+ constexpr int nfrags = MMQ_TILE_NE_K / tile_A::J;++ y += (threadIdx.y % ntx) * (tile_C::J * MMQ_TILE_Y_K);++ const int * x_qs = (const int *) x;+ const uint32_t * x_sc = (const uint32_t *) (x_qs + 2 * MMQ_TILE_NE_K);+ const int * y_qs = (const int *) y + 4;+ const uint32_t * y_sc = (const uint32_t *) y;++ // 2 threads per quad supply the packed scale register to the block_scale MMA,+ // see https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-block-scaling+ const int tidx_A = threadIdx.x / 4 + (threadIdx.x % 2) * 8;+ const int tidx_B = threadIdx.x / 4;+ const int i0 = (threadIdx.y / ntx) * rows_per_warp;++ tile_A A[ntx][nfrags];+ uint32_t scaleA[ntx][nfrags];++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int frag = 0; frag < nfrags; ++frag) {+ const int k0 = k00 + frag * tile_A::J;+ load_ldmatrix(A[n][frag], x_qs + (i0 + n * tile_A::I) * sram_stride + k0, sram_stride);+ scaleA[n][frag] = x_sc[(i0 + n * tile_A::I + tidx_A) * sram_stride + k0 / tile_A::J];+ }+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx * tile_C::J) {+ tile_B B[nfrags];+ uint32_t scaleB[nfrags];++#pragma unroll+ for (int frag = 0; frag < nfrags; ++frag) {+ const int k0 = frag * tile_B::J;+ load_generic(B[frag], y_qs + j0 * MMQ_TILE_Y_K + k0, MMQ_TILE_Y_K);+ scaleB[frag] = y_sc[(j0 + tidx_B) * MMQ_TILE_Y_K + frag];+ }++#pragma unroll+ for (int n = 0; n < ntx; ++n) {+ // accumulate in place into the output sum array+ tile_C & C = *reinterpret_cast<tile_C *>(sum + (j0 / tile_C::J + n) * tile_C::ne);+#pragma unroll+ for (int frag = 0; frag < nfrags; ++frag) {+ mma_block_scaled_fp4<type>(C, A[n][frag], B[frag], scaleA[n][frag], scaleB[frag]);+ }+ }+ }+}+diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cunew file mode 100644index 00000000..f754837d--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cu@@ -0,0 +1,522 @@+#include "common.cuh"+#include "mmq.cuh"+#include "quantize.cuh"+#include "mmid.cuh"++#include <cstdint>++static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream, const ggml_prec prec_src1) {+ switch (args.type_x) {+ case GGML_TYPE_Q1_0:+ mul_mat_q_case<GGML_TYPE_Q1_0, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q2_0:+ mul_mat_q_case<GGML_TYPE_Q2_0, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q4_0:+ mul_mat_q_case<GGML_TYPE_Q4_0, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q4_1:+ mul_mat_q_case<GGML_TYPE_Q4_1, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q5_0:+ mul_mat_q_case<GGML_TYPE_Q5_0, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q5_1:+ mul_mat_q_case<GGML_TYPE_Q5_1, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q8_0:+ mul_mat_q_case<GGML_TYPE_Q8_0, GGML_PREC_Q8>(ctx, args, stream);+ break;+// -----------------------------------------------------------------------+ case GGML_TYPE_Q2_K:+ mul_mat_q_case<GGML_TYPE_Q2_K, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q3_K:+ mul_mat_q_case<GGML_TYPE_Q3_K, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q4_K:+ mul_mat_q_case<GGML_TYPE_Q4_K, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q5_K:+ mul_mat_q_case<GGML_TYPE_Q5_K, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_Q6_K:+ mul_mat_q_case<GGML_TYPE_Q6_K, GGML_PREC_Q8>(ctx, args, stream);+ break;+// -----------------------------------------------------------------------+ case GGML_TYPE_IQ1_S:+ mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_IQ2_XXS:+ mul_mat_q_case<GGML_TYPE_IQ2_XXS, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_IQ2_XS:+ mul_mat_q_case<GGML_TYPE_IQ2_XS, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_IQ2_S:+ mul_mat_q_case<GGML_TYPE_IQ2_S, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_IQ3_XXS:+ mul_mat_q_case<GGML_TYPE_IQ3_XXS, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_IQ3_S:+ mul_mat_q_case<GGML_TYPE_IQ3_S, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_IQ4_XS:+ mul_mat_q_case<GGML_TYPE_IQ4_XS, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_IQ4_NL:+ mul_mat_q_case<GGML_TYPE_IQ4_NL, GGML_PREC_Q8>(ctx, args, stream);+ break;+// -----------------------------------------------------------------------+ case GGML_TYPE_MXFP4:+ // src1 at Q4 uses the native FP4 instructions, which are Blackwell-only+ if (prec_src1 == GGML_PREC_Q4) {+ mul_mat_q_case<GGML_TYPE_MXFP4, GGML_PREC_Q4>(ctx, args, stream);+ break;+ }+ mul_mat_q_case<GGML_TYPE_MXFP4, GGML_PREC_Q8>(ctx, args, stream);+ break;+ case GGML_TYPE_NVFP4:+ if (prec_src1 == GGML_PREC_Q4) {+ mul_mat_q_case<GGML_TYPE_NVFP4, GGML_PREC_Q4>(ctx, args, stream);+ break;+ }+ mul_mat_q_case<GGML_TYPE_NVFP4, GGML_PREC_Q8>(ctx, args, stream);+ break;+ default:+ GGML_ABORT("fatal error");+ }+}++// overrides the src1 precision requested by the graph, "auto" keeps the requested one+static ggml_prec ggml_cuda_mmq_get_prec_env() {+ const char * env_c = getenv("GGML_CUDA_MMQ_PREC");+ if (env_c == nullptr) {+ return GGML_PREC_UNDEFINED;+ }+ std::string env_cpp = env_c;+ for (char & c : env_cpp) {+ c = std::tolower(c);+ }+ if (env_cpp == "q4") {+ return GGML_PREC_Q4;+ }+ if (env_cpp == "q8") {+ return GGML_PREC_Q8;+ }+ if (env_cpp != "auto") {+ GGML_LOG_WARN("%s: Unknown value for GGML_CUDA_MMQ_PREC: '%s'. Available: 'q4', 'q8', 'auto'.\n", __func__, env_cpp.c_str());+ }+ return GGML_PREC_UNDEFINED;+}++// src1 is quantized to Q8_1 unless the FP4 types can use 4-bit activations, in which case they+// default to the native W4A4 instructions on Blackwell.+static ggml_prec ggml_cuda_mmq_get_prec_src1(const ggml_tensor * src0, const ggml_tensor * dst, const int cc) {+ static const ggml_prec prec_env = ggml_cuda_mmq_get_prec_env();++ ggml_prec prec = prec_env;+ if (prec == GGML_PREC_UNDEFINED) {+ prec = (ggml_prec) ggml_get_op_params_i32(dst, 3);+ }++ // Q4 only for the FP4 types on Blackwell+ GGML_ASSERT(prec == GGML_PREC_UNDEFINED || prec == GGML_PREC_Q8 || prec == GGML_PREC_Q4);+ const bool can_use_q4 = (src0->type == GGML_TYPE_NVFP4 || src0->type == GGML_TYPE_MXFP4) && blackwell_mma_available(cc);+ if (prec == GGML_PREC_Q8 || !can_use_q4) {+ return GGML_PREC_Q8;+ }+ return GGML_PREC_Q4;+}++void ggml_cuda_mul_mat_q(+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,+ int64_t ncols_max) {+ // FreeToken: bf16 and fp16 activations are quantized directly, except for the native fp4 paths; dst is fp32, bf16 or fp16+ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.++ GGML_TENSOR_BINARY_OP_LOCALS;++ cudaStream_t stream = ctx.stream();++ const int id = ggml_cuda_get_device();+ const int cc = ggml_cuda_info().devices[id].cc;+ const size_t smpbo = ggml_cuda_info().devices[id].smpbo;++ const size_t ts_src0 = ggml_type_size(src0->type);+ const size_t ts_src1 = ggml_type_size(src1->type);+ const size_t ts_dst = ggml_type_size(dst->type);++ GGML_ASSERT( nb00 == ts_src0);+ GGML_ASSERT( nb10 == ts_src1);+ GGML_ASSERT( nb0 == ts_dst);+ GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));++ const char * src0_d = (const char *) src0->data;+ const float * src1_d = (const float *) src1->data;+ const mmq_dst dst_d = {dst->data, dst->type};+ const nv_bfloat16 * src1_bf16 = src1->type == GGML_TYPE_BF16 ? (const nv_bfloat16 *) src1->data : nullptr;+ const half * src1_f16 = src1->type == GGML_TYPE_F16 ? (const half *) src1->data : nullptr;++ // If src0 is a temporary compute buffer, clear any potential padding.+ if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {+ const size_t size_data = ggml_nbytes(src0);+ const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);+ if (size_alloc > size_data) {+ GGML_ASSERT(ggml_is_contiguously_allocated(src0));+ GGML_ASSERT(!src0->view_src);+ CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));+ }+ }++ const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);++ const int64_t s01 = src0->nb[1] / ts_src0;+ const int64_t s1 = dst->nb[1] / ts_dst;+ const int64_t s02 = src0->nb[2] / ts_src0;+ const int64_t s2 = dst->nb[2] / ts_dst;+ const int64_t s03 = src0->nb[3] / ts_src0;+ const int64_t s3 = dst->nb[3] / ts_dst;++ const bool fallback = ggml_cuda_mmq_needs_fallback(ne01);++ const ggml_prec prec_src1 = ggml_cuda_mmq_get_prec_src1(src0, dst, cc);++ const bool use_native_fp4 = prec_src1 == GGML_PREC_Q4;+ GGML_ASSERT(!use_native_fp4 || src1->type == GGML_TYPE_F32);+ const size_t y_block_size = use_native_fp4 ? sizeof(block_fp4_mmq) : sizeof(block_q8_1_mmq);+ const size_t y_values_per_block = use_native_fp4 ? QK_FP4_MMQ : QK8_1_MMQ;++ int J_best = 0;+ int nthreads_best = 0;+ {+ int64_t ncols_opt = ne11;+ if (ids) {+ const int64_t n_expert_used = ids->ne[0];+ // a flat grid runs only the tiles that hold tokens, so it sizes them for the average expert+ ncols_opt = ncols_max > 0 ? ncols_max : (ne12*n_expert_used + ne02 - 1) / ne02;++ // Each expert only sees ne12*n_expert_used/ne02 tokens on average.+ // On RDNA3 and RDNA4 it is faster to pick the tile size against this value instead of ne12.+ if (GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_RDNA4(cc)) {+ ncols_opt = (ne12*n_expert_used + ne02 - 1) / ne02;+ }+ }++ int ntiles_J_best = INT_MAX;++ for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) {+ const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(src0->type, J, fallback, cc, prec_src1);+ if (config.type == GGML_TYPE_COUNT) {+ continue;+ }++ if (mmq_get_nbytes_shared(config, cc) > smpbo) {+ continue;+ }++ const int ntiles_x = (ncols_opt + config.J - 1) / config.J;++ if (ntiles_x < ntiles_J_best) {+ J_best = J;+ nthreads_best = config.nthreads;+ ntiles_J_best = ntiles_x;+ }+ }+ }+ GGML_ASSERT(J_best > 0);++ // A tile of size J can read in at most J - 1 extra columns.+ // For simplicity, round up the padding of a full tile to a multiple of the number of bytes that nthreads can load in parallel.+ const size_t src1_load_chunk_size = nthreads_best * sizeof(int);+ const size_t src1_q8_1_padding = ((J_best * sizeof(block_q8_1_mmq) + src1_load_chunk_size - 1) / src1_load_chunk_size)+ * src1_load_chunk_size;++ if (!ids) {+ const size_t nbytes_src1_q8_1 = ne13*ne12 * ne11*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);+ ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());+ if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) {+ src1_scale.alloc(ne13*ne12*ne11);+ }++ {+ const int64_t s11 = src1->nb[1] / ts_src1;+ const int64_t s12 = src1->nb[2] / ts_src1;+ const int64_t s13 = src1->nb[3] / ts_src1;+ if (use_native_fp4) {+ static constexpr size_t align_float8 = 32;+ const bool use_aligned_float8 = ggml_cuda_is_aligned(src1, align_float8);+ static_assert(sizeof(block_fp4_mmq) == 4 * sizeof(block_q8_1));+ quantize_mmq_fp4_cuda(src1_d, nullptr, src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13, ne10_padded,+ ne11, ne12, ne13, stream);++ } else if (src1_bf16) {+ quantize_mmq_q8_1_cuda(src1_bf16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,+ ne11, ne12, ne13, stream);+ } else if (src1_f16) {+ quantize_mmq_q8_1_cuda(src1_f16, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,+ ne11, ne12, ne13, stream);+ } else {+ quantize_mmq_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded,+ ne11, ne12, ne13, stream);+ }+ CUDA_CHECK(cudaGetLastError());+ }++ // Stride depends on quantization format+ const int64_t s12 = use_native_fp4 ?+ ne11 * ne10_padded * sizeof(block_fp4_mmq) / (QK_FP4_MMQ * sizeof(int)) :+ ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));+ const int64_t s13 = ne12*s12;++ const mmq_args args = {+ src0_d, src0->type, (const int *) src1_q8_1.ptr, nullptr, nullptr, dst_d,+ src0->type == GGML_TYPE_NVFP4 && use_native_fp4 ? src1_scale.ptr : nullptr,+ ne00, ne01, ne1, s01, ne11, s1,+ ne02, ne12, s02, s12, s2,+ ne03, ne13, s03, s13, s3,+ ne1, J_best};+ ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);+ return;+ }++ GGML_ASSERT(ne13 == 1);+ GGML_ASSERT(nb12 % nb11 == 0);+ GGML_ASSERT(nb2 % nb1 == 0);++ const int64_t n_expert_used = ids->ne[0];+ const int64_t ne_get_rows = ne12 * n_expert_used;+ GGML_ASSERT(ne1 == n_expert_used);++ ggml_cuda_pool_alloc<int32_t> ids_src1(ctx.pool(), ne_get_rows);+ ggml_cuda_pool_alloc<int32_t> ids_dst(ctx.pool(), ne_get_rows + J_best-1); // Needs to be padded for unconditional memory access.+ ggml_cuda_pool_alloc<int32_t> expert_bounds(ctx.pool(), ne02 + 1);++ // gate/up activations are broadcast across experts (ne11 == 1): quantize each token once and+ // scatter to its slots. ids_src1 then holds the inverse map (token slot -> compact row).+ const bool dedup_bcast = ne11 == 1 && n_expert_used > 1;++ {+ GGML_ASSERT(ids->nb[0] == ggml_element_size(ids));+ const int si1 = ids->nb[1] / ggml_element_size(ids);+ const int sis1 = nb12 / nb11;++ ggml_cuda_launch_mm_ids_helper((const int32_t *) ids->data, ids_src1.get(), ids_dst.get(), expert_bounds.get(),+ ne02, ne12, n_expert_used, ne11, si1, sis1, /*write_inverse =*/ dedup_bcast, stream);+ CUDA_CHECK(cudaGetLastError());+ }++ // Every expert's column tiles fit in ntx_flat grid columns, so a flat grid needs no host sync to size it.+ ggml_cuda_pool_alloc<int2> tile_map(ctx.pool());+ int ntx_flat = 0;+ if (ncols_max <= 0) {+ const int jt_none = (ne_get_rows + J_best - 1) / J_best;+ ntx_flat = jt_none + std::min<int64_t>(ne02, ne_get_rows);+ tile_map.alloc(ntx_flat);+ ggml_cuda_launch_mm_ids_tile_map(expert_bounds.get(), tile_map.get(), ne02, J_best, ntx_flat, jt_none, stream);+ CUDA_CHECK(cudaGetLastError());+ }++ const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block + src1_q8_1_padding;+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);+ ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());+ if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) {+ src1_scale.alloc(ne12*n_expert_used);+ }++ const int64_t ne11_flat = ne12*n_expert_used;+ const int64_t ne12_flat = 1;+ const int64_t ne13_flat = 1;++ {+ const int64_t s11 = src1->nb[1] / ts_src1;+ const int64_t s12 = src1->nb[2] / ts_src1;+ const int64_t s13 = src1->nb[3] / ts_src1;++ if (use_native_fp4) {+ static constexpr size_t align_float8 = 32;+ const bool use_aligned_float8 = ggml_cuda_is_aligned(src1, align_float8);+ if (dedup_bcast) {+ quantize_scatter_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10,+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);+ } else {+ quantize_mmq_fp4_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src1_scale.ptr, src0->type, use_aligned_float8, ne10, s11, s12, s13,+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);+ }+ } else if (dedup_bcast && src1_bf16) {+ quantize_scatter_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);+ } else if (dedup_bcast && src1_f16) {+ quantize_scatter_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);+ } else if (dedup_bcast) {+ quantize_scatter_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10,+ /*stride_token=*/s12, ne10_padded, ne12, ne11_flat, n_expert_used, stream);+ } else if (src1_bf16) {+ quantize_mmq_q8_1_cuda(src1_bf16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);+ } else if (src1_f16) {+ quantize_mmq_q8_1_cuda(src1_f16, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);+ } else {+ quantize_mmq_q8_1_cuda(src1_d, ids_src1.get(), src1_q8_1.get(), src0->type, ne10, s11, s12, s13,+ ne10_padded, ne11_flat, ne12_flat, ne13_flat, stream);+ }+ CUDA_CHECK(cudaGetLastError());+ }++ static_assert(QK_FP4_MMQ == 8 * QK_MXFP4, "QK_FP4_MMQ needs to be 8 * QK_MXFP4");+ const int64_t s12 = use_native_fp4 ? ne11 * ne10_padded * sizeof(block_fp4_mmq) / (QK_FP4_MMQ * sizeof(int)) :+ ne11 * ne10_padded * sizeof(block_q8_1) / (QK8_1 * sizeof(int));+ const int64_t s13 = ne12*s12;++ // Note that ne02 is used instead of ne12 because the number of y channels determines the z dimension of the CUDA grid.+ const mmq_args args = {+ src0_d, src0->type, (const int *) src1_q8_1.get(), ids_dst.get(), expert_bounds.get(), dst_d,+ src1_scale.ptr,+ ne00, ne01, ne_get_rows, s01, ne_get_rows, s1,+ ne02, ne02, s02, s12, s2,+ ne03, ne13, s03, s13, s3,+ ncols_max > 0 ? ncols_max : ne12, J_best, tile_map.ptr, ntx_flat};++ ggml_cuda_mul_mat_q_switch_type(ctx, args, stream, prec_src1);+}++bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts) {+#ifdef GGML_CUDA_FORCE_CUBLAS+ return false;+#endif // GGML_CUDA_FORCE_CUBLAS++ bool mmq_supported;++ switch (type) {+ case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+// -------------------------------------------------+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+// -------------------------------------------------+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ2_S:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ3_S:+ case GGML_TYPE_IQ4_XS:+ case GGML_TYPE_IQ4_NL:+// -------------------------------------------------+ case GGML_TYPE_MXFP4:+ case GGML_TYPE_NVFP4:+ mmq_supported = true;+ break;+ default:+ mmq_supported = false;+ break;+ }++ if (!mmq_supported) {+ return false;+ }++ // MMQ tiles require at least 48 KiB per-block shared memory; fall back to BLAS otherwise.+ {+ const int id = ggml_cuda_get_device();+ const size_t smpbo = ggml_cuda_info().devices[id].smpbo;+ if (smpbo < 48 * 1024) {+ return false;+ }+ }++ if (turing_mma_available(cc)) {+ return true;+ }++ if (ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_DP4A) {+ // for MoE, mmq is faster even without native dp4a+ // TODO: check if cards older than pascal might benefit from this as well+ return cc >= GGML_CUDA_CC_PASCAL && n_experts > 0;+ }++#ifdef GGML_CUDA_FORCE_MMQ+ return true;+#endif //GGML_CUDA_FORCE_MMQ++ if (GGML_CUDA_CC_IS_NVIDIA(cc)) {+ return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;+ }++ if (amd_mfma_available(cc)) {+ // As of ROCM 7.0 rocblas/tensile performs very poorly on CDNA3 and hipblaslt (via ROCBLAS_USE_HIPBLASLT)+ // performs better but is currently suffering from a crash on this architecture.+ // TODO: Revisit when hipblaslt is fixed on CDNA3+ if (GGML_CUDA_CC_IS_CDNA3(cc)) {+ return true;+ }+ if (n_experts > 64 || ne11 <= 128) {+ return true;+ }+ if (type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q5_0 || type == GGML_TYPE_Q5_1) {+ return true;+ }+ if (ne11 <= 256 && (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K)) {+ return true;+ }+ return false;+ }++ if (amd_wmma_available(cc)) {+ if (GGML_CUDA_CC_IS_RDNA3(cc)) {+ // High expert counts are almost always better on MMQ due to+ // the synchronization overhead in the cuBLAS/hipBLAS path:+ // https://github.com/ggml-org/llama.cpp/pull/18202+ if (n_experts >= 64) {+ return true;+ }++ // For some quantization types MMQ can have lower peak TOPS than hipBLAS+ // so it's only faster for sufficiently small batch sizes:+ switch (type) {+ case GGML_TYPE_Q2_K:+ return ne11 <= 128;+ case GGML_TYPE_Q6_K:+ return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ2_S:+ return GGML_CUDA_CC_IS_RDNA3_5(cc) || ne11 <= 128;+ default:+ return true;+ }+ }++ // For RDNA4 MMQ is consistently faster than dequantization + hipBLAS:+ // https://github.com/ggml-org/llama.cpp/pull/18537#issuecomment-3706422301+ return true;+ }++ // gfx900 (Vega 10), gfx909, and gfx90c lack native dp4a, losing to dequant + hipBLAS+ // for dense matrices; keep MMQ only for MoE, where the+ // hipBLAS path is much slower.+ if (cc == GGML_CUDA_CC_VEGA || GGML_CUDA_CC_IS_GCN_APU(cc)) {+ return n_experts > 0;+ }++ // MUSA: the MMQ kernels compute wrong values on PH1 (MTT S5000).+ if (cc == GGML_CUDA_CC_PH1) {+ return false;+ }++ return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuhnew file mode 100644index 00000000..596a54ea--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmq.cuh@@ -0,0 +1,1661 @@+#pragma once++#include "common.cuh"++#include <climits>+#include <cstdint>++#define MMQ_DP4A_MAX_BATCH_SIZE 64 // Max. batch size to use for dp4a MMQ kernels when FP16 tensor cores are available.+#define MMQ_ITER_K 256+#define MMQ_ITER_K_FP4 512+#define MMQ_NWARPS 8++// FreeToken: MMQ writes fp32, bf16 or fp16 results (dst_t); the sums stay fp32 until the store+template <typename T> static __device__ __forceinline__ T mmq_cast(float v);+template <> __device__ __forceinline__ float mmq_cast<float>(float v) { return v; }+template <> __device__ __forceinline__ nv_bfloat16 mmq_cast<nv_bfloat16>(float v) { return __float2bfloat16(v); }+template <> __device__ __forceinline__ half mmq_cast<half>(float v) { return __float2half(v); }++struct mmq_dst {+ void * p;+ ggml_type type;+};++typedef void (*ggml_cuda_mmq_load_tiles_t)(const char * __restrict__ x, int * x_tile, const int kbx0, const int i_max, const int stride);+typedef void (*ggml_cuda_mmq_vec_dot_t)(const int * __restrict__ x, const int * __restrict__ y, float * __restrict__ sum, const int k00);+typedef void (*ggml_cuda_mmq_write_back_t)(const float * __restrict__ sum, const int32_t * __restrict__ get_rows_to_sorted,+ float * __restrict__ dst, const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max);++enum mmq_q8_1_ds_layout {+ MMQ_Q8_1_DS_LAYOUT_D4,+ MMQ_Q8_1_DS_LAYOUT_DS4,+ MMQ_Q8_1_DS_LAYOUT_D2S6,+};++static constexpr int QK8_1_MMQ = 4*QK8_1;+static constexpr int QK_FP4_MMQ = 2*QK8_1_MMQ;++struct block_q8_1_mmq {+ // The y float data is converted to a data layout that can simply be copied to shared memory as a contiguous block.+ // The y float data is first grouped as blocks of 128 values.+ // These blocks are then treated as individual data values and transposed.+ //+ // To avoid shared memory bank conflicts each block is padded with 16 bytes.+ // This padding is also used to store block scales/partial sums.+ // The scales multiplied with the quantized data are equal to the unquantized values.+ // The partial sums are obtained by summing up a subgroup of the contained values (prior to quantization)+ // and are only needed for performance reasons.+ //+ // The exact data stored depends on the x data type.+ union {+ float d4[4]; // 1 32 bit scale per 32 values, stored as d0,d1,d2,d3+ half2 ds4[4]; // 1 16 bit scale + 1 16 bit partial sum per 32 values, stored as d0,s0,d1,s1,d2,s2,d3,s3+ half d2s6[8]; // 1 16 bit scale per 64 values + 1 16 bit partial sum per 16 values for the first 96 values,+ // stored as d0,d1,s1,s2,s3,s4,s5+ };+ int8_t qs[QK8_1_MMQ];+};++// this struct is used for fp4 data types (currently only used for Blackwell)+// mxfp4 has block size 32, each int32 of d4 contains 2 e8m0 scales in the lower 16 bits+// nvfp4 has block size 16, each int32 of d4 contains 4 ue4m3 scales+struct block_fp4_mmq {+ uint32_t d4[4];+ int8_t qs[QK_FP4_MMQ / 2];+};++static_assert(sizeof(block_q8_1_mmq) == QK8_1_MMQ + 4*sizeof(half2), "Unexpected block_q8_1_mmq size");+static_assert(sizeof(block_q8_1_mmq) == 4*sizeof(block_q8_1), "Unexpected block_q8_1_mmq size");+static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected block_fp4_mmq size");++static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) {+ switch (type_x) {+ case GGML_TYPE_Q1_0:+ case GGML_TYPE_Q2_0:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ return MMQ_Q8_1_DS_LAYOUT_DS4;+ case GGML_TYPE_Q5_0:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ case GGML_TYPE_Q5_1:+ return MMQ_Q8_1_DS_LAYOUT_DS4;+ case GGML_TYPE_Q8_0:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ case GGML_TYPE_MXFP4:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ case GGML_TYPE_NVFP4:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ case GGML_TYPE_Q2_K:+ return MMQ_Q8_1_DS_LAYOUT_D2S6;+ case GGML_TYPE_Q3_K:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ return MMQ_Q8_1_DS_LAYOUT_DS4;+ case GGML_TYPE_Q6_K:+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ2_S:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ3_S:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ case GGML_TYPE_IQ1_S:+ return MMQ_Q8_1_DS_LAYOUT_DS4;+ case GGML_TYPE_IQ4_XS:+ case GGML_TYPE_IQ4_NL:+ return MMQ_Q8_1_DS_LAYOUT_D4;+ default:+ GGML_ABORT("fatal error");+ }+}++struct tile_x_sizes {+ int qs;+ int dm;+ int sc;+};++// Decouple shared memory tile sizes from WARP_SIZE to allow for different warp sizes.+// The K dimension of the tiles has either,+// 1*MMQ_TILE_NE_K==32 (always for TILE_Y_K) or 2*MMQ_TILE_NE_K==64 (typically for TILE_X_K),+// 32 bit elements for the quantized data (does not include scales).+// In other words, the size of the quantized data in the K dimension is a multiple of MMQ_TILE_NE_K.+// The final tile size in K direction is padded to avoid shared memory bank conflicts,+// in terms of 32 bit elements that means K % 2 == 1 for dp4a or K % 8 == 4 for mma.+#define MMQ_TILE_NE_K 32++// block_q8_1_mmq has (128 8-bit ints == 32 32-bit ints + 4 32-bit scales)+#define MMQ_TILE_Y_K (MMQ_TILE_NE_K + MMQ_TILE_NE_K / QI8_1)+#define MMQ_TILE_Y_FP4_K MMQ_TILE_Y_K++enum ggml_cuda_mmq_sram_layout {+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0,+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1,+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K,+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K,+ GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K,+ GGML_CUDA_MMQ_SRAM_LAYOUT_FP4, // MXFP4 and NVFP4 on Blackwell.+ GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, // Generic NVFP4+};++static constexpr __host__ __device__ int ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_sram_layout sram_layout) {+ switch (sram_layout) {+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0:+ return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_0 + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1:+ return 2*MMQ_TILE_NE_K + 2*MMQ_TILE_NE_K/QI8_1 + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K:+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K:+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K:+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/QI6_K + MMQ_TILE_NE_K/8 + 7;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_FP4:+ return 2*MMQ_TILE_NE_K + 8 + 4;+ case GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4:+ return 2*MMQ_TILE_NE_K + MMQ_TILE_NE_K/2 + 4;+ default:+ return -1;+ }+}++static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) % 8 == 4, "Wrong padding.");+static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4) % 8 == 4, "Wrong padding.");++static_assert(ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_FP4) == ggml_cuda_mmq_get_sram_stride(GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1), "Wrong tile size for MXFP4");++// Config options for the MMQ kernel.+// Should not affect results, only speed/register pressure/shared memory use.+struct ggml_cuda_mmq_config {+ ggml_type type; // src0->type+ int nthreads; // Number of threads per CUDA block.+ int occupancy; // Targeted occupancy for the MMA kernel.+ int I; // SRAM tile width in src0->ne[1]/dst->ne[0] direction.+ int J; // SRAM tile width in src1->ne[1]/dst->ne[1] direction.+ ggml_cuda_mmq_sram_layout sram_layout; // SRAM tile length in src0->ne[0]/src1->ne[0] direction (physical 32 bit elements).+ int K_vram; // VRAM tile length in src0->ne[0]/src1->ne[0] direction (logical elements).+ bool stream_k; // Whether or not to use stream-k decomposition.+ bool fallback; // Whether a fallback for out-of-bounds check in src0->ne[1] direction is needed.++ constexpr __host__ __device__ ggml_cuda_mmq_config(+ ggml_type type, int nthreads, int occupancy, int I, int J, ggml_cuda_mmq_sram_layout sram_layout, int K_vram, bool stream_k, bool fallback) :+ type(type), nthreads(nthreads), occupancy(occupancy), I(I), J(J), sram_layout(sram_layout), K_vram(K_vram), stream_k(stream_k), fallback(fallback) {}++ constexpr __device__ int rows_per_warp() const {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ return 16;+#else+ return J >= 48 && J % 16 == 0 ? 32 : 16;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++ // TODO transition all combinations of GPUs and quantizations to the MMA data layout.+ __host__ int use_mma_data_layout(const int cc) const {+ if (amd_mfma_available(cc) || amd_wmma_available(cc) || turing_mma_available(cc)) {+ return true;+ }+ return false;+ }++ constexpr __device__ bool use_mma_data_layout() const {+#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)+ return true;+#else+ return false;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE)+ }++};++#define CASE(type_, nthreads_, occupancy_, I_, J_, sram_layout_, K_vram_, stream_k_, fallback_) \+ if (type == (type_) && J == (J_) && fallback == (fallback_)) { \+ static_assert((nthreads_) % 32 == 0 && (nthreads_) <= 512, "bad nthreads"); \+ static_assert( (occupancy_) <= 8, "bad occupancy"); \+ static_assert((I_) % 32 == 0, "bad I"); \+ static_assert((J_) % 8 == 0 && (J_) <= 128, "bad J"); \+ static_assert((K_vram_) % 256 == 0, "bad K_vram"); \+ return ggml_cuda_mmq_config((type_), (nthreads_), (occupancy_), (I_), (J_), (sram_layout_), (K_vram_), (stream_k_), (fallback_)); \+ } \++#include "mmq-config-pascal-older.cuh"+#include "mmq-config-pascal-dp4a.cuh"+#include "mmq-config-ampere.cuh"+#include "mmq-config-blackwell.cuh"++#include "mmq-config-gcn.cuh"+#include "mmq-config-cdna.cuh"+#include "mmq-config-rdna2.cuh"+#include "mmq-config-rdna3.cuh"+#include "mmq-config-rdna3-5.cuh"+#include "mmq-config-rdna4.cuh"++#undef CASE++static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ if (GGML_CUDA_CC_IS_AMD(cc)) {+ if (GGML_CUDA_CC_IS_GCN(cc)) {+ return ggml_cuda_mmq_get_config_gcn(type, J, fallback);+ }+ if (GGML_CUDA_CC_IS_CDNA(cc)) {+ return ggml_cuda_mmq_get_config_cdna(type, J, fallback);+ }+ if (GGML_CUDA_CC_IS_RDNA4(cc)) {+ return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);+ }+ if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {+ return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);+ }+ if (GGML_CUDA_CC_IS_RDNA3(cc)) { // covers RDNA 3.0+ return ggml_cuda_mmq_get_config_rdna3(type, J, fallback);+ }+ return ggml_cuda_mmq_get_config_rdna2(type, J, fallback);+ }+ if (blackwell_mma_available(cc)) {+ // only src1 at Q4 uses the native FP4 config, higher precisions keep src1 at Q8_1+ if (prec_src1 != GGML_PREC_Q4 && (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4)) {+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);+ }+ return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);+ }+ if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA) {+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);+ }+ if (ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_DP4A) {+ return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback);+ }+ return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback);+}++static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+#ifdef GGML_USE_HIP+#ifdef GCN+ return ggml_cuda_mmq_get_config_gcn(type, J, fallback);+#elif defined(CDNA)+ return ggml_cuda_mmq_get_config_cdna(type, J, fallback);+#elif defined(RDNA4)+ return ggml_cuda_mmq_get_config_rdna4(type, J, fallback);+#elif defined(RDNA3_5)+ return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback);+#elif defined(RDNA3)+ return ggml_cuda_mmq_get_config_rdna3(type, J, fallback);+#else+ return ggml_cuda_mmq_get_config_rdna2(type, J, fallback);+#endif // CDNA+#else+#ifdef BLACKWELL_MMA_AVAILABLE+ // only src1 at Q4 uses the native FP4 config, higher precisions keep src1 at Q8_1+ if (prec_src1 != GGML_PREC_Q4 && (type == GGML_TYPE_NVFP4 || type == GGML_TYPE_MXFP4)) {+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);+ }+ return ggml_cuda_mmq_get_config_blackwell(type, J, fallback);+#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA+ return ggml_cuda_mmq_get_config_ampere(type, J, fallback);+#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_DP4A+ return ggml_cuda_mmq_get_config_pascal_dp4a(type, J, fallback);+#else+ return ggml_cuda_mmq_get_config_pascal_older(type, J, fallback);+#endif // BLACKWELL_MMA_AVAILABLE+#endif // GGML_USE_HIP+ GGML_UNUSED_VARS(type, J, fallback, prec_src1);+}++static __host__ int ggml_cuda_mmq_get_type(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).type;+}++static constexpr __device__ int ggml_cuda_mmq_get_type(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type;+}++static constexpr __device__ int ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).nthreads;+}++static constexpr __device__ int ggml_cuda_mmq_get_occupancy(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).occupancy;+}++static __host__ int ggml_cuda_mmq_get_I(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).I;+}++static constexpr __device__ int ggml_cuda_mmq_get_I(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).I;+}++static __host__ int ggml_cuda_mmq_get_J(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).J;+}++static constexpr __device__ int ggml_cuda_mmq_get_J(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).J;+}++static __host__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).sram_layout;+}++static constexpr __device__ ggml_cuda_mmq_sram_layout ggml_cuda_mmq_get_sram_layout(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).sram_layout;+}++static __host__ int ggml_cuda_mmq_get_K_vram(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).K_vram;+}++static constexpr __device__ int ggml_cuda_mmq_get_K_vram(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).K_vram;+}++static __host__ bool ggml_cuda_mmq_get_stream_k(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).stream_k;+}++static constexpr __device__ bool ggml_cuda_mmq_get_stream_k(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).stream_k;+}++static __host__ int ggml_cuda_mmq_get_fallback(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1).fallback;+}++static constexpr __device__ int ggml_cuda_mmq_get_fallback(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).fallback;+}++// ---------------------------------------------------------------------------------------------++static __host__ int ggml_cuda_mmq_get_sram_stride(const ggml_type type, const int J, const bool fallback, const int cc, const ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, cc, prec_src1));+}++static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback, prec_src1));+}++static __host__ bool ggml_cuda_mmq_needs_fallback(const int64_t nrows_x) {+ return nrows_x % 128 != 0;+}++static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, int J, bool fallback, ggml_prec prec_src1) {+ return ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).rows_per_warp();+}++#define MMQ_DP4A_TXS_Q4_0 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_0 + I/QI4_0, 0}+#define MMQ_DP4A_TXS_Q4_1 tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_1 + I/QI4_1, 0}+#define MMQ_DP4A_TXS_Q8_0 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_0 + I/(QI8_0/2), 0}+#define MMQ_DP4A_TXS_Q8_0_16 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*4/QI8_0 + I/(QI8_0/4), 0}+#define MMQ_DP4A_TXS_Q8_1 tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K*2/QI8_1 + I/(QI8_1/2), 0}+#define MMQ_DP4A_TXS_Q2_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K + I, 0}+#define MMQ_DP4A_TXS_Q3_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I, I*MMQ_TILE_NE_K/8 + I/8}+#define MMQ_DP4A_TXS_Q4_K tile_x_sizes{I*MMQ_TILE_NE_K + I, I*MMQ_TILE_NE_K/QI4_K, I*MMQ_TILE_NE_K/8 + I/8}+#define MMQ_DP4A_TXS_Q5_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI5_K + I/QI5_K, I*MMQ_TILE_NE_K/8 + I/8}+#define MMQ_DP4A_TXS_Q6_K tile_x_sizes{I*MMQ_TILE_NE_K*2 + I, I*MMQ_TILE_NE_K/QI6_K + I/QI6_K, I*MMQ_TILE_NE_K/8 + I/8}++static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) {+ switch (type) {+ case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0;+ case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1;+ case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_Q5_1: return MMQ_DP4A_TXS_Q8_1;+ case GGML_TYPE_Q8_0: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_MXFP4: return MMQ_DP4A_TXS_Q8_1;+ case GGML_TYPE_NVFP4: return MMQ_DP4A_TXS_Q8_0_16;+ case GGML_TYPE_Q2_K: return MMQ_DP4A_TXS_Q2_K;+ case GGML_TYPE_Q3_K: return MMQ_DP4A_TXS_Q3_K;+ case GGML_TYPE_Q4_K: return MMQ_DP4A_TXS_Q4_K;+ case GGML_TYPE_Q5_K: return MMQ_DP4A_TXS_Q5_K;+ case GGML_TYPE_Q6_K: return MMQ_DP4A_TXS_Q6_K;+ case GGML_TYPE_IQ2_XXS: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;+ case GGML_TYPE_IQ2_S: return MMQ_DP4A_TXS_Q8_0_16;+ case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0;+ case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0;+ default: return tile_x_sizes{0, 0, 0};+ }+}++// FIXME temporary until all combinations of data types and GPUs can use the MMA data layout+static __host__ int ggml_cuda_mmq_get_nbytes_shared_x(const ggml_cuda_mmq_config & config, const int cc) {+ if (config.use_mma_data_layout(cc)) {+ return config.I * ggml_cuda_mmq_get_sram_stride(config.sram_layout) * 4;+ }+ const tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(config.type, config.I);+ return (txs.qs + txs.dm + txs.sc) * 4;+}++// ------------------------------------------------------------++#include "mmq-load-tiles.cuh"+#include "mmq-vec-dot.cuh"++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float> static __device__ __forceinline__ void ggml_cuda_mmq_write_back_dp4a(+ const float * __restrict__ sum, const int32_t * __restrict__ ids_dst, dst_t * __restrict__ dst,+ const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);++ const bool y_scale_used = y_scale != nullptr;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ if (j > j_max) {+ return;+ }++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += warp_size) {+ const int i = i0 + threadIdx.x;++ if (fallback && i > i_max) {+ continue;+ }++ if constexpr (type == GGML_TYPE_NVFP4) {+ if (y_scale_used) {+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);+ } else {+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);+ }+ } else {+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/nwarps) * (I/warp_size) + i0/warp_size]);+ GGML_UNUSED(y_scale_used);+ }+ }+ }+}++template<ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t = float>+static __device__ __forceinline__ void ggml_cuda_mmq_write_back_mma(+ const float * __restrict__ sum, const int * __restrict__ ids_dst, dst_t * __restrict__ dst,+ const float * __restrict__ y_scale, const int stride, const int i_max, const int j_max) {++#if defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ typedef tile<16, 16, int, DATA_LAYOUT_J_MAJOR> tile_C;+#else+ typedef tile<16, 8, int> tile_C;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ constexpr int rows_per_warp = ggml_cuda_mmq_get_rows_per_warp(type, J, fallback, prec_src1);+ constexpr int ntx = rows_per_warp/tile_C::I; // Number of x minitiles per warp.++ const int i0 = (threadIdx.y / ntx) * (ntx*tile_C::I);++ const bool y_scale_used = y_scale != nullptr;++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += ntx*tile_C::J) {+#pragma unroll+ for (int n = 0; n < ntx; ++n) {+#pragma unroll+ for (int l = 0; l < tile_C::ne; ++l) {+ const int j = j0 + (threadIdx.y % ntx) * tile_C::J + tile_C::get_j(l);++ if (j > j_max) {+ continue;+ }++ const int i = i0 + n*tile_C::I + tile_C::get_i(l);++ if (fallback && i > i_max) {+ continue;+ }++ if constexpr (type == GGML_TYPE_NVFP4) {+ if (y_scale_used) {+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(y_scale[j] * sum[(j0/tile_C::J + n)*tile_C::ne + l]);+ } else {+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);+ }+ } else {+ dst[ids_dst[j]*stride + i] = mmq_cast<dst_t>(sum[(j0/tile_C::J + n)*tile_C::ne + l]);+ GGML_UNUSED(y_scale_used);+ }+ }+ }+ }+}++// -------------------------------------------------------------------------------------------------------------------------------------++// TODO remove this struct and use ggml_cuda_mmq_sram_layout instead.+struct ggml_cuda_mmq_util_funcs {+ int vdr;+ ggml_cuda_mmq_load_tiles_t load_tiles;+ ggml_cuda_mmq_vec_dot_t vec_dot;+ ggml_cuda_mmq_write_back_t write_back;++ constexpr __host__ __device__ ggml_cuda_mmq_util_funcs(+ int vdr, ggml_cuda_mmq_load_tiles_t load_tiles, ggml_cuda_mmq_vec_dot_t vec_dot, ggml_cuda_mmq_write_back_t write_back) :+ vdr(vdr), load_tiles(load_tiles), vec_dot(vec_dot), write_back(write_back) {}+};++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>+static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_funcs() {+ if (!ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) {+ switch (type) {+ case GGML_TYPE_Q1_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q1_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q2_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q2_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q4_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q4_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q4_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q4_1:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q4_1_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q4_1_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q5_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q5_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q5_1:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q5_1_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q8_0:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q8_0_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_Q2_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q2_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q2_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q3_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q3_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q3_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q4_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q4_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q4_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q5_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q5_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q5_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q6_K:+ return ggml_cuda_mmq_util_funcs(+ VDR_Q6_K_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q6_K_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_IQ1_S:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ1_S_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ2_XXS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ2_XXS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ2_XS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ2_XS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ2_S:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ2_S_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ3_XXS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ3_XXS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ3_S:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ3_S_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ4_XS:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ4_XS_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ4_NL:+ return ggml_cuda_mmq_util_funcs(+ VDR_IQ4_NL_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_MXFP4:+ return ggml_cuda_mmq_util_funcs(+ VDR_MXFP4_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ case GGML_TYPE_NVFP4:+ return ggml_cuda_mmq_util_funcs(+ VDR_NVFP4_Q8_1_MMQ,+ ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback, prec_src1>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);+ default:+ return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);+ }+ }++// ---------------------------------------------------------------------------------------------++#ifdef BLACKWELL_MMA_AVAILABLE+ switch (type) {+ case GGML_TYPE_MXFP4:+ if (prec_src1 == GGML_PREC_Q4) {+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_mxfp4_fp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ }+ break;+ case GGML_TYPE_NVFP4:+ if (prec_src1 == GGML_PREC_Q4) {+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_nvfp4_nvfp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_fp4_fp4_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ }+ break;+ default:+ break;+ }+#endif // BLACKWELL_MMA_AVAILABLE++// ---------------------------------------------------------------------------------------------++ switch (type) {+ case GGML_TYPE_Q1_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q1_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q2_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q2_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q4_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q4_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_DS4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q4_1:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q4_1<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q5_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q5_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q5_1:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q5_1<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q8_0:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q8_0<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_Q2_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q2_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q2_K_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q3_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q3_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q4_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q4_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q5_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q5_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_Q6_K:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_q6_K<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q6_K_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_IQ1_S:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq1_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma<type, J, fallback>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ2_XXS:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq2_xxs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ2_XS:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq2_xs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ2_S:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq2_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ3_XXS:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ3_S:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq3_s<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ4_XS:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq4_xs<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_IQ4_NL:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_iq4_nl<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+// ---------------------------------------------------------------------------------------------+ case GGML_TYPE_MXFP4:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_mxfp4<type, J, fallback>,+ ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma<type, J, fallback, MMQ_Q8_1_DS_LAYOUT_D4>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ case GGML_TYPE_NVFP4:+ return ggml_cuda_mmq_util_funcs(+ -1,+ ggml_cuda_mmq_load_tiles_nvfp4<type, J, fallback, prec_src1>,+ ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback, prec_src1>,+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);+ default:+ return ggml_cuda_mmq_util_funcs(1, nullptr, nullptr, nullptr);+ }+}++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>+static constexpr __device__ int ggml_cuda_mmq_get_vdr() {+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().vdr;+}++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>+static constexpr __device__ ggml_cuda_mmq_load_tiles_t ggml_cuda_mmq_get_load_tiles() {+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().load_tiles;+}++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>+static constexpr __device__ ggml_cuda_mmq_vec_dot_t ggml_cuda_mmq_get_vec_dot() {+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().vec_dot;+}++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>+static constexpr __device__ ggml_cuda_mmq_write_back_t ggml_cuda_mmq_get_write_back() {+ return ggml_cuda_mmq_get_util_funcs<type, J, fallback, prec_src1>().write_back;+}++// ---------------------------------------------------------------------------------------------++template <ggml_type type, int J, bool fallback, bool fixup, ggml_prec prec_src1, typename dst_t>+static __device__ __forceinline__ void mul_mat_q_process_tile(+ const char * __restrict__ x, const int offset_x, const int * __restrict__ y,+ const int * __restrict__ ids_dst, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,+ const float * __restrict__ y_scale,+ const int stride_row_x, const int ncols_y, const int stride_col_dst,+ const int tile_x_max_i, const int tile_y_max_j, const int kb0_start, const int kb0_stop) {++ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;+ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);+ constexpr ggml_cuda_mmq_load_tiles_t load_tiles = ggml_cuda_mmq_get_load_tiles<type, J, fallback, prec_src1>();+ constexpr ggml_cuda_mmq_vec_dot_t vec_dot = ggml_cuda_mmq_get_vec_dot<type, J, fallback, prec_src1>();+ constexpr ggml_cuda_mmq_write_back_t write_back = ggml_cuda_mmq_get_write_back<type, J, fallback, prec_src1>();++ extern __shared__ int data_mul_mat_q[];+ int * tile_y = data_mul_mat_q + J;+ int * tile_x = tile_y + GGML_PAD(J*MMQ_TILE_Y_K, nwarps*warp_size);++#if defined(BLACKWELL_MMA_AVAILABLE)+ // FP4 tile stores 8 blocks. src1 above Q4 uses the generic+ // Q8_1 tile layout instead of the packed FP4 tile.+ constexpr int ne_block = ((type == GGML_TYPE_MXFP4 || type == GGML_TYPE_NVFP4) && prec_src1 == GGML_PREC_Q4) ?+ QK_FP4_MMQ : QK8_1_MMQ;+#else+ constexpr int ne_block = QK8_1_MMQ;+#endif // defined(BLACKWELL_MMA_AVAILABLE)++ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1);+ constexpr int blocks_per_iter = ITER_K / qk;++ float sum[J*I / (nwarps*warp_size)] = {0.0f};++ constexpr int sz = sizeof(block_q8_1_mmq) / sizeof(int);++ for (int kb0 = kb0_start; kb0 < kb0_stop; kb0 += blocks_per_iter) {+ load_tiles(x, tile_x, offset_x + kb0, tile_x_max_i, stride_row_x);+ {+ const int * by0 = y + ncols_y * (kb0 * qk / ne_block) * sz;+#pragma unroll+ for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {+ int l = l0 + threadIdx.y*warp_size + threadIdx.x;++ tile_y[l] = by0[l];+ }+ }++ __syncthreads();++ vec_dot(tile_x, tile_y, sum, 0);++ __syncthreads();++ {+ const int * by0 = y + ncols_y * ((kb0 * qk / ne_block) * sz + sz);+#pragma unroll+ for (int l0 = 0; l0 < J * MMQ_TILE_Y_K; l0 += nwarps * warp_size) {+ int l = l0 + threadIdx.y*warp_size + threadIdx.x;++ tile_y[l] = by0[l];+ }+ }++ __syncthreads();++ vec_dot(tile_x, tile_y, sum, MMQ_TILE_NE_K);++ __syncthreads();+ }++ if (fixup) {+ write_back(sum, ids_dst, tmp_fixup + blockIdx.x*(J*I), y_scale, I, I, J);+ } else if constexpr (std::is_same_v<dst_t, float>) {+ write_back(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);+ } else if constexpr (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).use_mma_data_layout()) {+ ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);+ } else {+ ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1, dst_t>(sum, ids_dst, dst, y_scale, stride_col_dst, tile_x_max_i, tile_y_max_j);+ }+}+++// A flat MoE grid runs only the (expert, column tile) pairs listed in tile_map, see ggml_cuda_launch_mm_ids_tile_map.+static __device__ __forceinline__ void mmq_flat_tile(const int2 * __restrict__ tile_map, int & zt, int & jt) {+ if (tile_map) {+ const int2 t = tile_map[jt];+ zt = t.x;+ jt = t.y;+ }+}++// The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, typename dst_t>+__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1))+static __global__ void mul_mat_q(+ const char * __restrict__ x, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,+ const int32_t * __restrict__ expert_bounds, const int2 * __restrict__ tile_map, dst_t * __restrict__ dst, float * __restrict__ tmp_fixup,+ const float * __restrict__ y_scale,+ const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst, const int stride_row_x, const int ncols_y, const int stride_col_dst,+ const uint3 channel_ratio, const uint3 nchannels_y, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,+ const uint3 sample_ratio, const uint3 nsamples_y, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,+ const uint3 ntx) {++ // Skip unused template specializations for faster compilation:+ if (ggml_cuda_mmq_get_config(type, J, fallback, prec_src1).type == GGML_TYPE_COUNT) {+ NO_DEVICE_CODE;+ return;+ }++ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / warp_size;+ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);++ const uint32_t nty = (nrows_x + I - 1) / I; // Number of tiles y++ // Initialize the ids for writing back data with just the index.+ // For regular matrix multiplications this is never changed.+ // For MoE the correct indices are loaded from ids_dst.+ extern __shared__ int ids_dst_shared[]; // Stored at beginning of shared memory.+#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {+ const int j = j0 + threadIdx.y*warp_size + threadIdx.x;++ if (j0 + nwarps*warp_size > J && j >= J) {+ break;+ }++ ids_dst_shared[j] = j;+ }+ __syncthreads();++ if constexpr (!ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {+ const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);+ const int wt = tmp2.x;+ int zt = tmp2.y;+ int jt = blockIdx.y;+ const int it = blockIdx.x;+ mmq_flat_tile(tile_map, zt, jt);++ // Defaults for regular matrix multiplication:+ int col_low = 0;+ int col_high = ncols_dst;+ int col_diff = ncols_dst;+ int offset_y = wt*stride_sample_y + zt*stride_channel_y;+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;+ int offset_y_scale;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;+ } else {+ GGML_UNUSED(offset_y_scale);+ }++ if (ids_dst) {+ col_low = expert_bounds[zt + 0];+ col_high = expert_bounds[zt + 1];+ col_diff = col_high - col_low;++ offset_y = 0;+ offset_dst = 0;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale = 0;+ }++ if (jt*J >= col_diff) {+ return;+ }++ // __syncthreads(); // There is no previous tile that could cause a race condition.+#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {+ const int j = j0 + threadIdx.y*warp_size + threadIdx.x;++ if (j0 + nwarps*warp_size > J && j >= J) {+ break;+ }++ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];+ }+ __syncthreads();+ }++ offset_y += (col_low + jt*J)*(sizeof(block_q8_1_mmq)/sizeof(int));+ offset_dst += it*I;+ const float * y_scale_tile = nullptr;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale += col_low + jt*J;+ y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;+ }++ const int tile_x_max_i = nrows_x - it*I - 1;+ const int tile_y_max_j = col_diff - jt*J - 1;++ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;++ constexpr bool fixup = false;+ mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>+ (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,+ stride_row_x, ncols_y, stride_col_dst,+ tile_x_max_i, tile_y_max_j, 0, blocks_per_ne00.z);+ return;+ }++ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1);+ constexpr int blocks_per_iter = ITER_K / qk;++ // kbc == k block continuous, current index in continuous ijk space.+ int kbc = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;+ int kbc_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;++ kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter;+ kbc_stop -= fastmodulo(kbc_stop, blocks_per_ne00) % blocks_per_iter;++ // kb0 == k index when doing the matrix multiplication for an output tile.+ int kb0_start = fastmodulo(kbc, blocks_per_ne00);+ int kb0_stop = min(blocks_per_ne00.z, uint32_t(kb0_start + kbc_stop - kbc));+ while (kbc < kbc_stop && kb0_stop == int(blocks_per_ne00.z)) {+ int tmp = fastdiv(kbc, blocks_per_ne00);+ uint2 tmp2 = fast_div_modulo(tmp, ntx);+ int jt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nchannels_y);+ int zt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nsamples_y);+ const int wt = tmp2.y;+ const int it = tmp2.x;+ mmq_flat_tile(tile_map, zt, jt);++ // Defaults for regular matrix multiplication:+ int col_low = 0;+ int col_high = ncols_dst;+ int col_diff = ncols_dst;+ int offset_y = wt*stride_sample_y + zt*stride_channel_y;+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;+ int offset_y_scale;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;+ } else {+ GGML_UNUSED(offset_y_scale);+ }++ if (ids_dst) {+ col_low = expert_bounds[zt + 0];+ col_high = expert_bounds[zt + 1];+ col_diff = col_high - col_low;++ offset_y = 0;+ offset_dst = 0;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale = 0;+ }++ if (jt*J >= col_diff) {+ kbc += blocks_per_ne00.z;+ kbc -= fastmodulo(kbc, blocks_per_ne00);++ kb0_start = 0;+ kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc));++ continue;+ }++ __syncthreads();+#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {+ const int j = j0 + threadIdx.y*warp_size + threadIdx.x;++ if (j0 + nwarps*warp_size > J && j >= J) {+ break;+ }++ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];+ }+ __syncthreads();+ }++ offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));+ offset_dst += it*I;+ const float * y_scale_tile = nullptr;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale += col_low + jt * J;+ y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;+ }++ const int tile_x_max_i = nrows_x - it*I - 1;+ const int tile_y_max_j = col_diff - jt*J - 1;++ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;++ constexpr bool fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.+ mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>+ (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,+ stride_row_x, ncols_y, stride_col_dst,+ tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);++ kbc += blocks_per_ne00.z;+ kbc -= fastmodulo(kbc, blocks_per_ne00);++ kb0_start = 0;+ kb0_stop = min(blocks_per_ne00.z, uint32_t(kbc_stop - kbc));+ }++ if (kbc >= kbc_stop) {+ return;+ }++ int tmp = fastdiv(kbc, blocks_per_ne00);+ uint2 tmp2 = fast_div_modulo(tmp, ntx);+ int jt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nchannels_y);+ int zt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nsamples_y);+ const int wt = tmp2.y;+ const int it = tmp2.x;+ mmq_flat_tile(tile_map, zt, jt);++ // Defaults for regular matrix multiplication:+ int col_low = 0;+ int col_high = ncols_dst;+ int col_diff = ncols_dst;+ int offset_y = wt*stride_sample_y + zt*stride_channel_y;+ int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;+ int offset_y_scale;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;+ } else {+ GGML_UNUSED(offset_y_scale);+ }++ if (ids_dst) {+ col_low = expert_bounds[zt + 0];+ col_high = expert_bounds[zt + 1];+ col_diff = col_high - col_low;++ offset_y = 0;+ offset_dst = 0;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale = 0;+ }++ if (jt*J >= col_diff) {+ return;+ }++ // The memory layout for the fixup buffer is always contiguous, therefore reset ids:+ __syncthreads();+#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps*warp_size) {+ const int j = j0 + threadIdx.y*warp_size + threadIdx.x;++ if (j0 + nwarps*warp_size > J && j >= J) {+ break;+ }++ ids_dst_shared[j] = j;+ }+ __syncthreads();+ }++ offset_y += (col_low + jt * J) * (sizeof(block_q8_1_mmq) / sizeof(int));+ offset_dst += it*I;+ const float * y_scale_tile = nullptr;+ if constexpr (type == GGML_TYPE_NVFP4) {+ offset_y_scale += col_low + jt * J;+ y_scale_tile = y_scale ? y_scale + offset_y_scale : nullptr;+ }++ const int tile_x_max_i = nrows_x - it*I - 1;+ const int tile_y_max_j = col_diff - jt*J - 1;++ const int offset_x = fastdiv(wt, sample_ratio)*stride_sample_x + fastdiv(zt, channel_ratio)*stride_channel_x + it*I*stride_row_x;++ constexpr bool fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.+ mul_mat_q_process_tile<type, J, fallback, fixup, prec_src1>+ (x, offset_x, y + offset_y, ids_dst_shared, dst + offset_dst, tmp_fixup, y_scale_tile,+ stride_row_x, ncols_y, stride_col_dst,+ tile_x_max_i, tile_y_max_j, kb0_start, kb0_stop);+}++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>+__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1)/2, 1)+static __global__ void mul_mat_q_stream_k_fixup(+ const int32_t * __restrict__ ids_dst, const int32_t * __restrict__ expert_bounds, float * __restrict__ dst,+ float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst,+ const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y,+ const int stride_sample_dst, const uint3 ntx) {+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / 2) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);+ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1);+ constexpr int blocks_per_iter = ITER_K / qk;++ float sum[J / nwarps] = {0.0f};+ const int i = blockIdx.y*warp_size + threadIdx.x;++ const int nty = (nrows_x + I - 1) / I;++ const int bidx0 = blockIdx.x;++ // kbc == k block continuous, current index in continuous ijk space.+ int kbc0 = int64_t(blockIdx.x) *(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;+ int kbc0_stop = int64_t(blockIdx.x + 1)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;++ kbc0 -= fastmodulo(kbc0, blocks_per_ne00) % blocks_per_iter;+ kbc0_stop -= fastmodulo(kbc0_stop, blocks_per_ne00) % blocks_per_iter;++ const bool did_not_have_any_data = kbc0 == kbc0_stop;+ const bool wrote_beginning_of_tile = fastmodulo(kbc0, blocks_per_ne00) == 0;+ const bool did_not_write_last = fastdiv(kbc0, blocks_per_ne00) == fastdiv(kbc0_stop, blocks_per_ne00) && fastmodulo(kbc0_stop, blocks_per_ne00) != 0;+ if (did_not_have_any_data || wrote_beginning_of_tile || did_not_write_last) {+ return;+ }++ bool any_fixup = false;++ // Iterate over previous blocks and sum up partial sums written to fixup buffer.+ // All CUDA blocks that get here must have a previous block that needs a fixup.+ int bidx = bidx0 - 1;+ int kbc_stop = kbc0;+ while(true) {+ int kbc = int64_t(bidx)*(nsamples_y.z*nchannels_y.z*ntx.z*nty*blocks_per_ne00.z) / gridDim.x;+ kbc -= fastmodulo(kbc, blocks_per_ne00) % blocks_per_iter;++ if (kbc == kbc_stop) { // Did not have any data.+ bidx--;+ kbc_stop = kbc;+ continue;+ }++ any_fixup = true;+++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ sum[j0/nwarps] += tmp_last_tile[bidx*(J*I) + j*I + i];+ }++ // If this block started in a previous tile we are done and don't need to combine additional partial results.+ if (fastmodulo(kbc, blocks_per_ne00) == 0 || fastdiv(kbc, blocks_per_ne00) < fastdiv(kbc0, blocks_per_ne00)) {+ break;+ }+ bidx--;+ kbc_stop = kbc;+ }++ if (!any_fixup) {+ return;+ }++ int tmp = fastdiv(kbc0, blocks_per_ne00);+ uint2 tmp2 = fast_div_modulo(tmp, ntx);+ const int jt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nchannels_y);+ const int zt = tmp2.y;+ tmp = tmp2.x;+ tmp2 = fast_div_modulo(tmp, nsamples_y);+ const int wt = tmp2.y;+ const int it = tmp2.x;++ if (!ids_dst) {+ const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;+ dst += offset_dst;++ const int i_max = nrows_x - it*I - 1;+ const int j_max = ncols_dst - jt*J - 1;+ if (fallback && i > i_max) {+ return;+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ if (j > j_max) {+ return;+ }++ dst[j*stride_col_dst + i] += sum[j0/nwarps];+ }+ return;+ }++ __shared__ int ids_dst_shared[J];+ const int col_low = expert_bounds[zt + 0];+ const int col_high = expert_bounds[zt + 1];+ const int col_diff = col_high - col_low;++ for (int j = threadIdx.y*warp_size + threadIdx.x; j < J; j += nwarps*warp_size) {+ ids_dst_shared[j] = ids_dst[col_low + jt*J + j];+ }+ __syncthreads();++ const int offset_dst = it*I;+ dst += offset_dst;++ const int i_max = nrows_x - it*I - 1;+ const int j_max = col_diff - jt*J - 1;+ if (fallback && i > i_max) {+ return;+ }++#pragma unroll+ for (int j0 = 0; j0 < J; j0 += nwarps) {+ const int j = j0 + threadIdx.y;++ if (j > j_max) {+ return;+ }++ dst[ids_dst_shared[j]*stride_col_dst + i] += sum[j0/nwarps];+ }+}++// FreeToken: copies an fp32 result with dst's layout into dst+template <typename dst_t>+static __global__ void mmq_store_dst(const float * __restrict__ src, dst_t * __restrict__ dst, const int nrows, const int stride) {+ const int64_t col = blockIdx.x;+ const int row = blockIdx.y*blockDim.x + threadIdx.x;+ if (row < nrows) {+ dst[col*stride + row] = mmq_cast<dst_t>(src[col*stride + row]);+ }+}++struct mmq_args {+ const char * x; ggml_type type_x; const int * y; const int32_t * ids_dst; const int32_t * expert_bounds; mmq_dst dst;+ const float * y_scale;+ int64_t ncols_x; int64_t nrows_x; int64_t ncols_dst; int64_t stride_row_x; int64_t ncols_y; int64_t nrows_dst;+ int64_t nchannels_x; int64_t nchannels_y; int64_t stride_channel_x; int64_t stride_channel_y; int64_t stride_channel_dst;+ int64_t nsamples_x; int64_t nsamples_y; int64_t stride_sample_x; int64_t stride_sample_y; int64_t stride_sample_dst;+ int64_t ncols_max;+ int J_best; // Tile width in ne11(dense)/ne12(MoE) direction to use for optimal performance.+ const int2 * tile_map = nullptr; // Flat MoE grid: the (expert, column tile) of each of the ntx_flat grid columns.+ int ntx_flat = 0;+};++static size_t mmq_get_nbytes_shared(const ggml_cuda_mmq_config & config, const int cc) {+ const size_t nbs_ids = config.J*sizeof(int);+ const size_t nbs_x = ggml_cuda_mmq_get_nbytes_shared_x(config, cc);+ const size_t nbs_y = config.J * (sizeof(block_q8_1_mmq));+ return nbs_ids + nbs_x + GGML_PAD(nbs_y, config.nthreads*sizeof(int));+}++template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>+static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {+ const int id = ggml_cuda_get_device();+ const int cc = ggml_cuda_info().devices[id].cc;+ const int nsm = ggml_cuda_info().devices[id].nsm;+ const int warp_size = ggml_cuda_info().devices[id].warp_size;++ const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc, prec_src1);+ GGML_ASSERT(config.nthreads % warp_size == 0);+ const int nwarps = config.nthreads / warp_size;+ const int nbytes_shared = mmq_get_nbytes_shared(config, cc);++ const dim3 block_dims(warp_size, nwarps, 1);++ // a flat MoE grid puts all experts into one channel of ntx_flat column tiles+ const int64_t nchannels_grid = args.tile_map ? 1 : args.nchannels_y;+ const int nty = (args.nrows_x + config.I - 1) / config.I;+ const int ntx = args.tile_map ? args.ntx_flat : (args.ncols_max + config.J - 1) / config.J;+ const int ntzw = nchannels_grid * args.nsamples_y;+ const dim3 block_nums_xy_tiling(nty, ntx, ntzw);++ GGML_ASSERT(args.nchannels_y % args.nchannels_x == 0);+ GGML_ASSERT(args.nsamples_y % args.nsamples_x == 0);+ const int channel_ratio = args.nchannels_y / args.nchannels_x;+ const int sample_ratio = args.nsamples_y / args.nsamples_x;++ const uint3 blocks_per_ne00_fd = init_fastdiv_values(args.ncols_x / ggml_cuda_type_traits<type>::qk);+ const uint3 ntx_fd = init_fastdiv_values(ntx);+ const uint3 nchannels_y_fd = init_fastdiv_values(nchannels_grid);+ const uint3 nsamples_y_fd = init_fastdiv_values(args.nsamples_y);+ const uint3 channel_ratio_fd = init_fastdiv_values(channel_ratio);+ const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);++ const auto run = [&](auto * dst, const dim3 & block_nums, float * tmp_fixup) {+ using dst_t = std::remove_pointer_t<decltype(dst)>;+ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, false, prec_src1, dst_t>), nbytes_shared);+ CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, true, prec_src1, dst_t>), nbytes_shared);+ mul_mat_q<type, J, fallback, prec_src1, dst_t><<<block_nums, block_dims, nbytes_shared, stream>>>+ (args.x, args.y, args.ids_dst, args.expert_bounds, args.tile_map, dst, tmp_fixup, args.y_scale,+ blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,+ channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,+ sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,+ ntx_fd);+ };++ // FreeToken: the fixup kernel adds partial sums into an fp32 dst, so a bf16 dst that needs it is computed in fp32 first.+ // An fp16 dst always is, unless GGML_MMQ_F16_DST builds the fp16 kernels.+ const auto launch = [&](const dim3 & block_nums, const bool fixup, float * tmp_fixup) {+ bool via_f32 = fixup && args.dst.type != GGML_TYPE_F32;+#ifndef GGML_MMQ_F16_DST+ via_f32 = via_f32 || args.dst.type == GGML_TYPE_F16;+#endif // GGML_MMQ_F16_DST+ ggml_cuda_pool_alloc<float> dst_f32(ctx.pool(id));+ float * dst_float = args.dst.type == GGML_TYPE_F32 ? (float *) args.dst.p : nullptr;+ if (via_f32) {+ GGML_ASSERT(args.ids_dst || (args.nchannels_y == 1 && args.nsamples_y == 1));+ dst_float = dst_f32.alloc(args.ncols_dst * args.nrows_dst);+ }++ if (dst_float) {+ run(dst_float, block_nums, tmp_fixup);+ } else if (args.dst.type == GGML_TYPE_BF16) {+ run((nv_bfloat16 *) args.dst.p, block_nums, tmp_fixup);+ }+#ifdef GGML_MMQ_F16_DST+ else {+ run((half *) args.dst.p, block_nums, tmp_fixup);+ }+#endif // GGML_MMQ_F16_DST++ if (fixup) {+ CUDA_CHECK(cudaGetLastError());+ const dim3 block_nums_fixup(block_nums.x, config.I/warp_size, 1);+ const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);+ mul_mat_q_stream_k_fixup<type, J, fallback, prec_src1><<<block_nums_fixup, block_dims_fixup, 0, stream>>>+ (args.ids_dst, args.expert_bounds, dst_float, tmp_fixup, blocks_per_ne00_fd, args.nrows_x, args.ncols_dst,+ args.nrows_dst, nchannels_y_fd, args.stride_channel_dst, nsamples_y_fd, args.stride_sample_dst,+ ntx_fd);+ }++ if (via_f32) {+ CUDA_CHECK(cudaGetLastError());+ const dim3 block_nums_store(args.ncols_dst, (args.nrows_x + 255) / 256, 1);+ if (args.dst.type == GGML_TYPE_BF16) {+ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (nv_bfloat16 *) args.dst.p, args.nrows_x, args.nrows_dst);+ } else {+ mmq_store_dst<<<block_nums_store, 256, 0, stream>>>(dst_float, (half *) args.dst.p, args.nrows_x, args.nrows_dst);+ }+ }+ };++ if (!config.stream_k) {+ launch(block_nums_xy_tiling, false, nullptr);+ return;+ }++ // For the stream-k kernel it is possible to run it with tiling by setting the number of CUDA blocks equal to the number of tiles.+ // This is worthwhile if the efficiency of tiling is high and skipping the fixup kernel is more important.+ const int ntiles_dst = ntx * nty * ntzw;+ const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;+ const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);+ // The fixup kernel does not read tile_map, so a flat MoE grid always runs one tile per block.+ const dim3 block_nums_stream_k(args.tile_map || (GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90) ? ntiles_dst : nsm, 1, 1);++ GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.++ const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0;++ ggml_cuda_pool & pool = ctx.pool(id);+ ggml_cuda_pool_alloc<float> tmp_fixup(pool);+ if (fixup_needed) {+ tmp_fixup.alloc(block_nums_stream_k.x * config.J*config.I);+ }++ launch(block_nums_stream_k, fixup_needed, tmp_fixup.ptr);+}++template <ggml_type type, bool fallback, ggml_prec prec_src1>+void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {+ switch (args.J_best) {+ case 8:+ launch_mul_mat_q<type, 8, fallback, prec_src1>(ctx, args, stream);+ break;+ case 16:+ launch_mul_mat_q<type, 16, fallback, prec_src1>(ctx, args, stream);+ break;+ case 24:+ launch_mul_mat_q<type, 24, fallback, prec_src1>(ctx, args, stream);+ break;+ case 32:+ launch_mul_mat_q<type, 32, fallback, prec_src1>(ctx, args, stream);+ break;+ case 40:+ launch_mul_mat_q<type, 40, fallback, prec_src1>(ctx, args, stream);+ break;+ case 48:+ launch_mul_mat_q<type, 48, fallback, prec_src1>(ctx, args, stream);+ break;+ case 56:+ launch_mul_mat_q<type, 56, fallback, prec_src1>(ctx, args, stream);+ break;+ case 64:+ launch_mul_mat_q<type, 64, fallback, prec_src1>(ctx, args, stream);+ break;+ case 72:+ launch_mul_mat_q<type, 72, fallback, prec_src1>(ctx, args, stream);+ break;+ case 80:+ launch_mul_mat_q<type, 80, fallback, prec_src1>(ctx, args, stream);+ break;+ case 88:+ launch_mul_mat_q<type, 88, fallback, prec_src1>(ctx, args, stream);+ break;+ case 96:+ launch_mul_mat_q<type, 96, fallback, prec_src1>(ctx, args, stream);+ break;+ case 104:+ launch_mul_mat_q<type, 104, fallback, prec_src1>(ctx, args, stream);+ break;+ case 112:+ launch_mul_mat_q<type, 112, fallback, prec_src1>(ctx, args, stream);+ break;+ case 120:+ launch_mul_mat_q<type, 120, fallback, prec_src1>(ctx, args, stream);+ break;+ case 128:+ launch_mul_mat_q<type, 128, fallback, prec_src1>(ctx, args, stream);+ break;+ default:+ fprintf(stderr, "J_best=%d\n", args.J_best);+ GGML_ABORT("fatal error");+ }+}++template <ggml_type type, ggml_prec prec_src1>+void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) {+ if (ggml_cuda_mmq_needs_fallback(args.nrows_x)) {+ constexpr bool fallback = true;+ mul_mat_q_switch_J<type, fallback, prec_src1>(ctx, args, stream);+ } else {+ constexpr bool fallback = false;+ mul_mat_q_switch_J<type, fallback, prec_src1>(ctx, args, stream);+ }+}++#define DECL_MMQ_CASE(type) \+ template void mul_mat_q_case<type, GGML_PREC_Q8>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \++// FP4 variant: uses native FP4 MMA instead of keeping src1 at Q8_1.+#define DECL_MMQ_CASE_W4A4(type) \+ template void mul_mat_q_case<type, GGML_PREC_Q4>(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \++extern DECL_MMQ_CASE(GGML_TYPE_Q1_0);+extern DECL_MMQ_CASE(GGML_TYPE_Q2_0);+extern DECL_MMQ_CASE(GGML_TYPE_Q4_0);+extern DECL_MMQ_CASE(GGML_TYPE_Q4_1);+extern DECL_MMQ_CASE(GGML_TYPE_Q5_0);+extern DECL_MMQ_CASE(GGML_TYPE_Q5_1);+extern DECL_MMQ_CASE(GGML_TYPE_Q8_0);+// -----------------------------------------+extern DECL_MMQ_CASE(GGML_TYPE_Q2_K);+extern DECL_MMQ_CASE(GGML_TYPE_Q3_K);+extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);+extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);+extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);+// -----------------------------------------+extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);+extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S);+extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);+extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);+extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);+extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);+// -----------------------------------------+extern DECL_MMQ_CASE(GGML_TYPE_MXFP4);+extern DECL_MMQ_CASE(GGML_TYPE_NVFP4);+extern DECL_MMQ_CASE_W4A4(GGML_TYPE_MXFP4);+extern DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4);++// -------------------------------------------------------------------------------------------------------------------------++// FreeToken: under ids, ncols_max > 0 bounds the tokens any one expert gets and sizes the tile grid; otherwise the grid is flat, see ggml_cuda_launch_mm_ids_tile_map+void ggml_cuda_mul_mat_q(+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,+ int64_t ncols_max = -1);++bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t n_experts);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cunew file mode 100644index 00000000..f3a3ccc6--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cu@@ -0,0 +1,1626 @@+#include "mmvq.cuh"+#include "convert.cuh"+#include "quantize.cuh"+#include "unary.cuh"+#include "vecdotq.cuh"++#include <cstdint>+#include <type_traits>++// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel+// has little exposed latency left to hide and the extra requests cost more than they save.+// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031+#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK+// returns true only for those quants that benefit from prefetch and false otherwise+static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_MXFP4:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ case GGML_TYPE_IQ1_M:+ case GGML_TYPE_IQ4_NL:+ case GGML_TYPE_IQ4_XS:+ return true;+ default:+ return false;+ }+}++static __device__ __forceinline__ void mmvq_prefetch_l2(const void * p) {+ asm volatile("prefetch.global.L2 [%0];" :: "l"(p));+}+#endif++typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs);++static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q1_0: return vec_dot_q1_0_q8_1;+ case GGML_TYPE_Q2_0: return vec_dot_q2_0_q8_1;+ case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1;+ case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1;+ case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1;+ case GGML_TYPE_Q5_1: return vec_dot_q5_1_q8_1;+ case GGML_TYPE_Q8_0: return vec_dot_q8_0_q8_1;+ case GGML_TYPE_MXFP4: return vec_dot_mxfp4_q8_1;+ case GGML_TYPE_NVFP4: return vec_dot_nvfp4_q8_1;+ case GGML_TYPE_Q2_K: return vec_dot_q2_K_q8_1;+ case GGML_TYPE_Q3_K: return vec_dot_q3_K_q8_1;+ case GGML_TYPE_Q4_K: return vec_dot_q4_K_q8_1;+ case GGML_TYPE_Q5_K: return vec_dot_q5_K_q8_1;+ case GGML_TYPE_Q6_K: return vec_dot_q6_K_q8_1;+ case GGML_TYPE_IQ2_XXS: return vec_dot_iq2_xxs_q8_1;+ case GGML_TYPE_IQ2_XS: return vec_dot_iq2_xs_q8_1;+ case GGML_TYPE_IQ2_S: return vec_dot_iq2_s_q8_1;+ case GGML_TYPE_IQ3_XXS: return vec_dot_iq3_xxs_q8_1;+ case GGML_TYPE_IQ1_S: return vec_dot_iq1_s_q8_1;+ case GGML_TYPE_IQ1_M: return vec_dot_iq1_m_q8_1;+ case GGML_TYPE_IQ4_NL: return vec_dot_iq4_nl_q8_1;+ case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1;+ case GGML_TYPE_IQ3_S: return vec_dot_iq3_s_q8_1;+ default: return nullptr;+ }+}++static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) {+ switch (type) {+ case GGML_TYPE_Q1_0: return VDR_Q1_0_Q8_1_MMVQ;+ case GGML_TYPE_Q2_0: return VDR_Q2_0_Q8_1_MMVQ;+ case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ;+ case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ;+ case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ;+ case GGML_TYPE_Q5_1: return VDR_Q5_1_Q8_1_MMVQ;+ case GGML_TYPE_Q8_0: return VDR_Q8_0_Q8_1_MMVQ;+ case GGML_TYPE_MXFP4: return VDR_MXFP4_Q8_1_MMVQ;+ case GGML_TYPE_NVFP4: return VDR_NVFP4_Q8_1_MMVQ;+ case GGML_TYPE_Q2_K: return VDR_Q2_K_Q8_1_MMVQ;+ case GGML_TYPE_Q3_K: return VDR_Q3_K_Q8_1_MMVQ;+ case GGML_TYPE_Q4_K: return VDR_Q4_K_Q8_1_MMVQ;+ case GGML_TYPE_Q5_K: return VDR_Q5_K_Q8_1_MMVQ;+ case GGML_TYPE_Q6_K: return VDR_Q6_K_Q8_1_MMVQ;+ case GGML_TYPE_IQ2_XXS: return VDR_IQ2_XXS_Q8_1_MMVQ;+ case GGML_TYPE_IQ2_XS: return VDR_IQ2_XS_Q8_1_MMVQ;+ case GGML_TYPE_IQ2_S: return VDR_IQ2_S_Q8_1_MMVQ;+ case GGML_TYPE_IQ3_XXS: return VDR_IQ3_XXS_Q8_1_MMVQ;+ case GGML_TYPE_IQ3_S: return VDR_IQ3_S_Q8_1_MMVQ;+ case GGML_TYPE_IQ4_NL: return VDR_IQ4_NL_Q8_1_MMVQ;+ case GGML_TYPE_IQ4_XS: return VDR_IQ4_XS_Q8_1_MMVQ;+ default: return 1;+ }+}++enum mmvq_parameter_table_id {+ MMVQ_PARAMETERS_GENERIC = 0,+ MMVQ_PARAMETERS_TURING,+ MMVQ_PARAMETERS_GCN,+ MMVQ_PARAMETERS_RDNA2,+ MMVQ_PARAMETERS_RDNA3_0,+ MMVQ_PARAMETERS_RDNA4,+ MMVQ_PARAMETERS_GB10+};++static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {+#if defined(RDNA4)+ return MMVQ_PARAMETERS_RDNA4;+#elif defined(RDNA3_0)+ return MMVQ_PARAMETERS_RDNA3_0;+#elif defined(RDNA2) || defined(RDNA3_5)+ return MMVQ_PARAMETERS_RDNA2;+#elif defined(GCN) || defined(CDNA)+ return MMVQ_PARAMETERS_GCN;+#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE+ return MMVQ_PARAMETERS_TURING;+#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK+ return MMVQ_PARAMETERS_GB10;+#else+ return MMVQ_PARAMETERS_GENERIC;+#endif+}++static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {+ if (GGML_CUDA_CC_IS_RDNA4(cc)) {+ return MMVQ_PARAMETERS_RDNA4;+ }+ if (GGML_CUDA_CC_IS_RDNA3_0(cc)) {+ return MMVQ_PARAMETERS_RDNA3_0;+ }+ if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) {+ return MMVQ_PARAMETERS_RDNA2;+ }+ if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) {+ return MMVQ_PARAMETERS_GCN;+ }+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) {+ return MMVQ_PARAMETERS_TURING;+ }+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) {+ return MMVQ_PARAMETERS_GB10;+ }+ return MMVQ_PARAMETERS_GENERIC;+}++// Per-architecture maximum batch size for which MMVQ should be used for MUL_MAT_ID.+// Returns a value <= MMVQ_MAX_BATCH_SIZE. Default is MMVQ_MAX_BATCH_SIZE.+// Check https://github.com/ggml-org/llama.cpp/pull/20905#issuecomment-4145835627 for details++static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_pascal_older(ggml_type type) {+ switch (type) {+ case GGML_TYPE_IQ1_S: return 6;+ case GGML_TYPE_IQ1_M: return 6;+ case GGML_TYPE_IQ2_S: return 4;+ case GGML_TYPE_IQ2_XS: return 5;+ case GGML_TYPE_IQ2_XXS: return 5;+ case GGML_TYPE_IQ3_S: return 4;+ case GGML_TYPE_IQ3_XXS: return 4;+ case GGML_TYPE_IQ4_NL: return 6;+ case GGML_TYPE_IQ4_XS: return 5;+ case GGML_TYPE_MXFP4: return 4;+ case GGML_TYPE_NVFP4: return 4;+ case GGML_TYPE_Q2_K: return 4;+ case GGML_TYPE_Q3_K: return 4;+ case GGML_TYPE_Q4_0: return 6;+ case GGML_TYPE_Q4_1: return 6;+ case GGML_TYPE_Q4_K: return 5;+ case GGML_TYPE_Q5_0: return 6;+ case GGML_TYPE_Q5_1: return 6;+ case GGML_TYPE_Q5_K: return 5;+ case GGML_TYPE_Q6_K: return 4;+ case GGML_TYPE_Q8_0: return 4;+ default: return MMVQ_MAX_BATCH_SIZE;+ }+}++static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_turing_plus(ggml_type type) {+ switch (type) {+ case GGML_TYPE_IQ2_S: return 7;+ case GGML_TYPE_IQ3_S: return 6;+ case GGML_TYPE_IQ3_XXS: return 7;+ case GGML_TYPE_MXFP4: return 7;+ case GGML_TYPE_NVFP4: return 8;+ case GGML_TYPE_Q2_K: return 7;+ case GGML_TYPE_Q3_K: return 5;+ default: return MMVQ_MAX_BATCH_SIZE;+ }+}++static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_gcn(ggml_type type) {+ switch (type) {+ case GGML_TYPE_IQ1_S: return 5;+ case GGML_TYPE_IQ1_M: return 5;+ case GGML_TYPE_IQ2_S: return 4;+ case GGML_TYPE_IQ2_XS: return 4;+ case GGML_TYPE_IQ2_XXS: return 4;+ case GGML_TYPE_IQ3_S: return 4;+ case GGML_TYPE_IQ3_XXS: return 4;+ case GGML_TYPE_IQ4_NL: return 6;+ case GGML_TYPE_IQ4_XS: return 4;+ case GGML_TYPE_Q2_K: return 4;+ case GGML_TYPE_Q3_K: return 4;+ case GGML_TYPE_Q4_0: return 5;+ case GGML_TYPE_Q4_1: return 5;+ case GGML_TYPE_Q4_K: return 4;+ case GGML_TYPE_Q5_K: return 4;+ case GGML_TYPE_Q6_K: return 4;+ case GGML_TYPE_Q8_0: return 4;+ default: return MMVQ_MAX_BATCH_SIZE;+ }+}++static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_cdna(ggml_type type) {+ switch (type) {+ case GGML_TYPE_IQ2_S: return 5;+ case GGML_TYPE_IQ2_XS: return 5;+ case GGML_TYPE_IQ2_XXS: return 5;+ case GGML_TYPE_IQ3_S: return 4;+ case GGML_TYPE_IQ3_XXS: return 5;+ default: return MMVQ_MAX_BATCH_SIZE;+ }+}++static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna1_rdna2(ggml_type type) {+ switch (type) {+ case GGML_TYPE_IQ2_S: return 4;+ case GGML_TYPE_IQ2_XS: return 4;+ case GGML_TYPE_IQ2_XXS: return 4;+ case GGML_TYPE_IQ3_S: return 4;+ case GGML_TYPE_IQ3_XXS: return 4;+ case GGML_TYPE_Q2_K: return 7;+ case GGML_TYPE_Q3_K: return 4;+ case GGML_TYPE_Q4_K: return 5;+ case GGML_TYPE_Q5_K: return 6;+ case GGML_TYPE_Q6_K: return 5;+ default: return MMVQ_MAX_BATCH_SIZE;+ }+}++static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna3(ggml_type type) {+ switch (type) {+ case GGML_TYPE_IQ1_S: return 6;+ case GGML_TYPE_IQ1_M: return 6;+ case GGML_TYPE_IQ2_S: return 4;+ case GGML_TYPE_IQ2_XS: return 4;+ case GGML_TYPE_IQ2_XXS: return 4;+ case GGML_TYPE_IQ3_S: return 4;+ case GGML_TYPE_IQ3_XXS: return 4;+ case GGML_TYPE_IQ4_NL: return 6;+ case GGML_TYPE_IQ4_XS: return 6;+ case GGML_TYPE_Q4_K: return 4;+ case GGML_TYPE_Q5_K: return 4;+ case GGML_TYPE_Q6_K: return 4;+ default: return MMVQ_MAX_BATCH_SIZE;+ }+}++static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type type) {+ switch (type) {+ case GGML_TYPE_IQ1_S: return 7;+ case GGML_TYPE_IQ1_M: return 7;+ case GGML_TYPE_IQ2_S: return 4;+ case GGML_TYPE_IQ2_XS: return 4;+ case GGML_TYPE_IQ2_XXS: return 4;+ case GGML_TYPE_IQ3_S: return 4;+ case GGML_TYPE_IQ3_XXS: return 4;+ case GGML_TYPE_IQ4_NL: return 7;+ case GGML_TYPE_IQ4_XS: return 5;+ case GGML_TYPE_MXFP4: return 5;+ case GGML_TYPE_NVFP4: return 5;+ case GGML_TYPE_Q3_K: return 4;+ case GGML_TYPE_Q4_0: return 7;+ case GGML_TYPE_Q4_1: return 7;+ case GGML_TYPE_Q4_K: return 4;+ case GGML_TYPE_Q5_0: return 7;+ case GGML_TYPE_Q5_1: return 7;+ case GGML_TYPE_Q5_K: return 5;+ case GGML_TYPE_Q6_K: return 5;+ case GGML_TYPE_Q8_0: return 7;+ default: return MMVQ_MAX_BATCH_SIZE;+ }+}++// Host function: returns the max batch size for the current arch+type at runtime.+int get_mmvq_mmid_max_batch(ggml_type type, int cc) {+ // NVIDIA: P100, Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID.+ if (GGML_CUDA_CC_IS_NVIDIA(cc)) {+ if (cc == GGML_CUDA_CC_PASCAL || cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) {+ return MMVQ_MAX_BATCH_SIZE;+ }+ if (cc >= GGML_CUDA_CC_TURING) {+ return get_mmvq_mmid_max_batch_turing_plus(type);+ }+ return get_mmvq_mmid_max_batch_pascal_older(type);+ }++ // AMD+ if (GGML_CUDA_CC_IS_AMD(cc)) {+ if (GGML_CUDA_CC_IS_RDNA4(cc)) {+ return get_mmvq_mmid_max_batch_rdna4(type);+ }+ if (GGML_CUDA_CC_IS_RDNA3(cc)) {+ return get_mmvq_mmid_max_batch_rdna3(type);+ }+ if (GGML_CUDA_CC_IS_RDNA1(cc) || GGML_CUDA_CC_IS_RDNA2(cc)) {+ return get_mmvq_mmid_max_batch_rdna1_rdna2(type);+ }+ if (GGML_CUDA_CC_IS_CDNA(cc)) {+ return get_mmvq_mmid_max_batch_cdna(type);+ }+ if (GGML_CUDA_CC_IS_GCN(cc)) {+ return get_mmvq_mmid_max_batch_gcn(type);+ }+ }+ return MMVQ_MAX_BATCH_SIZE;+}++bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {+ if (!ggml_is_quantized(type)) {+ return false;+ }+ // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner.+ // Only list quant-types MMQ supports, others would fall back to cuBLAS.+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) {+ switch (type) { // tuned on RTX 4090+ case GGML_TYPE_Q2_K:+ return ne11 <= 4;+ case GGML_TYPE_Q3_K:+ return ne11 <= 6;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) {+ switch (type) { // tuned on RTX 5090+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ return ne11 <= 5;+ case GGML_TYPE_Q5_K:+ return ne11 <= 6;+ case GGML_TYPE_Q6_K:+ return ne11 <= 7;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {+ switch (type) { // tuned on DGX Spark GB10+ case GGML_TYPE_Q2_K:+ return ne11 <= 6;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ORIN) {+ switch (type) { // tuned for Jetson Orin+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ return ne11 <= 1;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_VOLTA) {+ switch (type) {+ case GGML_TYPE_Q2_K:+ return ne11 <= 4;+ case GGML_TYPE_Q3_K:+ return ne11 <= 6;+ case GGML_TYPE_Q4_K:+ return ne11 <= 5;+ case GGML_TYPE_Q5_K:+ return ne11 <= 6;+ case GGML_TYPE_Q6_K:+ return ne11 <= 7;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ if (GGML_CUDA_CC_IS_CDNA(cc)) {+ if (GGML_CUDA_CC_IS_CDNA1(cc)) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ return ne11 <= 7;+ case GGML_TYPE_Q5_1:+ return ne11 <= 7;+ case GGML_TYPE_Q8_0:+ return ne11 <= 6;+ case GGML_TYPE_Q2_K:+ return ne11 <= 4;+ case GGML_TYPE_Q3_K:+ return ne11 <= 3;+ case GGML_TYPE_Q4_K:+ return ne11 <= 2;+ case GGML_TYPE_Q5_K:+ return ne11 <= 3;+ case GGML_TYPE_Q6_K:+ return ne11 <= 4;+ case GGML_TYPE_IQ1_S:+ return ne11 <= 5;+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ3_S:+ case GGML_TYPE_IQ4_XS:+ return ne11 <= 6;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ switch (type) { // tuned for CDNA2+ case GGML_TYPE_Q2_K:+ return ne11 <= 5;+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ return ne11 <= 3;+ case GGML_TYPE_Q6_K:+ return ne11 <= 5;+ default:+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+ }+ }+ return ne11 <= MMVQ_MAX_BATCH_SIZE;+}++// Device constexpr: returns the max batch size for the current arch+type at compile time.+template <ggml_type type>+static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() {+#if defined(RDNA4)+ return get_mmvq_mmid_max_batch_rdna4(type);+#elif defined(RDNA3)+ return get_mmvq_mmid_max_batch_rdna3(type);+#elif defined(RDNA2) || defined(RDNA1)+ return get_mmvq_mmid_max_batch_rdna1_rdna2(type);+#elif defined(CDNA)+ return get_mmvq_mmid_max_batch_cdna(type);+#elif defined(GCN)+ return get_mmvq_mmid_max_batch_gcn(type);+#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_PASCAL || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE)+ return MMVQ_MAX_BATCH_SIZE;+#elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING+ return get_mmvq_mmid_max_batch_turing_plus(type);+#else+ return get_mmvq_mmid_max_batch_pascal_older(type);+#endif+}++static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_dst, mmvq_parameter_table_id table_id, bool small_k = false, bool halve_iters = false) {+ if (table_id == MMVQ_PARAMETERS_GENERIC) {+ switch (ncols_dst) {+ case 1:+ case 2:+ case 3:+ case 4:+ return 4;+ case 5:+ case 6:+ case 7:+ case 8:+ return 2;+ default:+ return 1;+ }+ } else if (table_id == MMVQ_PARAMETERS_GCN) {+ switch (ncols_dst) {+ case 1:+ case 2:+ case 3:+ case 4:+ return 2;+ case 5:+ case 6:+ case 7:+ case 8:+ default:+ return 1;+ }+ }+ if (table_id == MMVQ_PARAMETERS_RDNA4) {+ // nwarps=8 benefits types with simple vec_dot on RDNA4 (ncols_dst=1).+ // Types with complex vec_dot (Q3_K, IQ2_*, IQ3_*) regress due to register+ // pressure and lookup table contention at higher thread counts.+ if (ncols_dst == 1) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ case GGML_TYPE_IQ4_NL:+ case GGML_TYPE_IQ4_XS:+ return 8;+ default:+ return 1;+ }+ }+ return 1;+ }+ if (table_id == MMVQ_PARAMETERS_RDNA3_0) {+ // RDNA3 (W7900): stricter whitelist than RDNA4.+ // Q2_K / Q5_K / IQ4_XS regress in full quant sweeps.+ if (ncols_dst == 1) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ return 8;+ case GGML_TYPE_Q6_K:+ return 2;+ case GGML_TYPE_IQ4_NL:+ return 8;+ default:+ return 1;+ }+ }+ return 1;+ }+ if (table_id == MMVQ_PARAMETERS_TURING) {+ if (ncols_dst == 1) {+ switch (type) {+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ return 2;+ default:+ return 4;+ }+ }+ switch (ncols_dst) {+ case 2:+ case 3:+ case 4:+ return 4;+ case 5:+ case 6:+ case 7:+ case 8:+ return 2;+ default:+ return 1;+ }+ }+ if (table_id == MMVQ_PARAMETERS_GB10) {+ const int generic = calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC);+ // Only worth the wider block when it actually retires the K loop in half the trips (Observation)+ if (ncols_dst == 1 && !small_k && halve_iters) {+ switch (type) {+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ case GGML_TYPE_IQ4_NL:+ return 2 * generic;+ default:+ break;+ }+ }+ return generic;+ }+ return 1;+}++static constexpr __host__ __device__ int calc_rows_per_block(int ncols_dst, int table_id, bool small_k = false, int nwarps = 1) {+ if (table_id == MMVQ_PARAMETERS_GENERIC || table_id == MMVQ_PARAMETERS_GCN || table_id == MMVQ_PARAMETERS_TURING || table_id == MMVQ_PARAMETERS_GB10) {+ switch (ncols_dst) {+ case 1:+ return small_k ? nwarps : 1;+ case 2:+ case 3:+ case 4:+ case 5:+ case 6:+ case 7:+ case 8:+ return 2;+ default:+ return 1;+ }+ }+ return 1;+}++// FreeToken: dst_t is float, nv_bfloat16 or half+template <ggml_type type, int ncols_dst, bool has_fusion, bool small_k = false, bool halve_iters = false, typename dst_t = float>+__launch_bounds__(calc_nwarps(type, ncols_dst, get_device_table_id(), small_k, halve_iters)*ggml_cuda_get_physical_warp_size(), 1)+static __global__ void mul_mat_vec_q(+ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst_ptr,+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,+ uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,+ const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,+ const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,+ const uint32_t ids_stride) {+ const void * GGML_CUDA_RESTRICT vx = vx_ptr;+ const void * GGML_CUDA_RESTRICT vy = vy_ptr;+ const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;+ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;++ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int qi = ggml_cuda_type_traits<type>::qi;+ constexpr int vdr = get_vdr_mmvq(type);+ constexpr mmvq_parameter_table_id table_id = get_device_table_id();+ constexpr int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);+ constexpr int rows_per_cuda_block = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();++ constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);++ const int tid = warp_size*threadIdx.y + threadIdx.x;+ const int row0 = rows_per_cuda_block*blockIdx.x;+ const int blocks_per_row_x = ncols_x / qk;+ constexpr int blocks_per_iter = vdr * nwarps*warp_size / qi;++ const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1;+ const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y;+ if (shared_expert) {+ vx = fusion.shared_up;+ if constexpr (std::is_same_v<dst_t, float>) {+ dst = fusion.shared_dst;+ }+ stride_col_dst = fusion.shared_stride_col_dst;+ }++ uint32_t channel_x;+ uint32_t channel_y;+ uint32_t sample_dst;++ ggml_cuda_pdl_sync();+ channel_x = shared_expert ? 0 : ncols_dst == 1 && ids ? ids[channel_dst] : fastdiv(channel_dst, channel_ratio);+ channel_y = ncols_dst == 1 && ids ? fastmodulo(channel_dst, nchannels_y) : channel_dst;+ sample_dst = blockIdx.z;++ const uint32_t sample_x = fastdiv(sample_dst, sample_ratio);+ const uint32_t sample_y = sample_dst;++ bool use_gate = false;+ bool use_bias = false;+ bool use_gate_bias = false;+ bool use_scale = false;+ bool use_gate_scale = false;+ [[maybe_unused]] const void * vgate = nullptr;+ const float * x_bias = nullptr;+ const float * gate_bias = nullptr;+ const float * x_scale = nullptr;+ const float * gate_scale = nullptr;+ ggml_glu_op active_glu;+ float glu_limit = 0.0f;++ if constexpr (has_fusion) {+ use_gate = fusion.gate != nullptr;+ use_bias = fusion.x_bias != nullptr;+ use_gate_bias = fusion.gate_bias != nullptr && use_gate;+ vgate = shared_expert ? fusion.shared_gate : fusion.gate;+ x_bias = (const float *) fusion.x_bias;+ gate_bias = (const float *) fusion.gate_bias;+ active_glu = fusion.glu_op;+ glu_limit = fusion.glu_limit;+ if constexpr (type == GGML_TYPE_NVFP4) {+ use_scale = fusion.x_scale != nullptr;+ use_gate_scale = fusion.gate_scale != nullptr && use_gate;+ x_scale = (const float *) fusion.x_scale;+ gate_scale = (const float *) fusion.gate_scale;+ }+ }+++ [[maybe_unused]] float x_biases[ncols_dst] = { 0.0f };+ [[maybe_unused]] float gate_biases[ncols_dst] = { 0.0f };+ [[maybe_unused]] float x_scales = 1.0f;+ [[maybe_unused]] float gate_scales = 1.0f;+ if constexpr (has_fusion) {+ // 1. Hide latency by prefetching bias, gates and scales here+ // 2. load only on threads that won't die after partial sum calculation+ const uint32_t channel_bias = ids ? channel_x : channel_dst;+ if (threadIdx.x < rows_per_cuda_block && threadIdx.y == 0 &&+ (rows_per_cuda_block == 1 || uint32_t(row0 + threadIdx.x) < stride_col_dst)) {+ if (use_bias) {+ x_bias = x_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ x_biases[j] = x_bias[j * stride_col_dst + threadIdx.x];+ }+ }+ if (use_gate_bias) {+ gate_bias = gate_bias + sample_dst * stride_sample_dst + channel_bias * stride_channel_dst + row0;+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+ gate_biases[j] = gate_bias[j * stride_col_dst + threadIdx.x];+ }+ }+ if constexpr (type == GGML_TYPE_NVFP4) {+ if (use_scale) {+ x_scales = x_scale[ids ? channel_x : 0];+ }+ if (use_gate_scale) {+ gate_scales = gate_scale[ids ? channel_x : 0];+ }+ }+ }+ }++ // partial sum for each thread+ float tmp[ncols_dst][rows_per_cuda_block] = {{0.0f}};+ float tmp_gate[ncols_dst][rows_per_cuda_block] = {{0.0f}};++ const block_q8_1 * y = ((const block_q8_1 *) vy) + sample_y*stride_sample_y + channel_y*stride_channel_y;+ const int kbx_offset = sample_x*stride_sample_x + channel_x*stride_channel_x + row0*stride_row_x;++ for (int kbx = tid / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {+ const int kby = kbx * (qk/QK8_1); // y block index that aligns with kbx++ // x block quant index when casting the quants to int+ const int kqs = vdr * (tid % (qi/vdr));++#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK+ // start the next iterations' weight loads early+ if constexpr (mmvq_should_prefetch(type)) {+ constexpr int pf_dist = 2; // loop iterations, not blocks+ const int kbx_pf = kbx + pf_dist*blocks_per_iter;+ if (kbx_pf < blocks_per_row_x) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+ const size_t off = (size_t)(kbx_offset + i*stride_row_x + kbx_pf) * ggml_cuda_type_traits<type>::bs;+ mmvq_prefetch_l2((const char *) vx + off);+ if constexpr (has_fusion) {+ if (use_gate) {+ mmvq_prefetch_l2((const char *) vgate + off);+ }+ }+ }+ }+ }+#endif++#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+ tmp[j][i] += vec_dot_q_cuda(+ vx, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);+ if constexpr (has_fusion) {+ if (use_gate) {+ tmp_gate[j][i] += vec_dot_q_cuda(+ vgate, &y[j*stride_col_y + kby], kbx_offset + i*stride_row_x + kbx, kqs);+ }+ }+ }+ }+ }++ __shared__ float tmp_shared[nwarps-1 > 0 ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];+ [[maybe_unused]] __shared__ float tmp_shared_gate[(has_fusion && (nwarps-1 > 0)) ? nwarps-1 : 1][ncols_dst][rows_per_cuda_block][warp_size];++ if (threadIdx.y > 0) {+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+ tmp_shared[threadIdx.y-1][j][i][threadIdx.x] = tmp[j][i];+ if constexpr (has_fusion) {+ if (use_gate) {+ tmp_shared_gate[threadIdx.y-1][j][i][threadIdx.x] = tmp_gate[j][i];+ }+ }+ }+ }+ }+ __syncthreads();+ if (threadIdx.y > 0) {+ return;+ }++ dst += sample_dst*stride_sample_dst + channel_dst*stride_channel_dst + row0;++ // sum up partial sums and write back result+#pragma unroll+ for (int j = 0; j < ncols_dst; ++j) {+#pragma unroll+ for (int i = 0; i < rows_per_cuda_block; ++i) {+#pragma unroll+ for (int l = 0; l < nwarps-1; ++l) {+ tmp[j][i] += tmp_shared[l][j][i][threadIdx.x];+ if constexpr (has_fusion) {+ if (use_gate) {+ tmp_gate[j][i] += tmp_shared_gate[l][j][i][threadIdx.x];+ }+ }+ }+ tmp[j][i] = warp_reduce_sum<warp_size>(tmp[j][i]);+ if constexpr (has_fusion) {+ if (use_gate) {+ tmp_gate[j][i] = warp_reduce_sum<warp_size>(tmp_gate[j][i]);+ }+ }++ if (threadIdx.x == i && (rows_per_cuda_block == 1 || uint32_t(row0 + i) < stride_col_dst)) {+ float result = tmp[j][i];+ if constexpr (has_fusion) {+ if constexpr (type == GGML_TYPE_NVFP4) {+ result *= x_scales;+ }+ result += x_biases[j];+ if (use_gate) {+ float gate_value = tmp_gate[j][i];+ if constexpr (type == GGML_TYPE_NVFP4) {+ gate_value *= gate_scales;+ }+ gate_value += gate_biases[j];+ switch (active_glu) {+ case GGML_GLU_OP_SWIGLU:+ result *= ggml_cuda_op_silu_single(gate_value);+ break;+ case GGML_GLU_OP_GEGLU:+ result *= ggml_cuda_op_gelu_single(gate_value);+ break;+ case GGML_GLU_OP_SWIGLU_OAI:+ result = ggml_cuda_op_swiglu_oai_single(gate_value, result);+ break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);+ break;+ default:+ result = result * gate_value;+ break;+ }+ }+ }+ dst[j*stride_col_dst + i] = ggml_cuda_cast<dst_t>(result);+ }+ }+ }++ if constexpr (!has_fusion) {+ GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, active_glu, glu_limit, gate_bias, x_bias, x_scale, gate_scale, tmp_gate);+ }+ if constexpr (type != GGML_TYPE_NVFP4) {+ GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale, x_scales, gate_scales);+ }+}++// Dedicated MoE multi-token kernel.+// Grid: (ceil(nrows_x / c_rows_per_block), nchannels_dst)+// Block: (warp_size, ncols_dst) - each warp handles one token independently.+// No shared memory reduction needed since each warp works alone.+template <ggml_type type, int c_rows_per_block, bool has_fusion = false, typename dst_t = float>+__launch_bounds__(get_mmvq_mmid_max_batch_for_device<type>()*ggml_cuda_get_physical_warp_size(), 1)+static __global__ void mul_mat_vec_q_moe(+ const void * vx_ptr, const void * vy_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion,+ dst_t * dst_ptr,+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,+ const uint32_t stride_row_x, const uint32_t stride_col_y, uint32_t stride_col_dst,+ const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,+ const uint32_t ncols_dst, const uint32_t ids_stride) {+ const void * GGML_CUDA_RESTRICT vx = vx_ptr;+ const void * GGML_CUDA_RESTRICT vy = vy_ptr;+ const int32_t * GGML_CUDA_RESTRICT ids = ids_ptr;+ dst_t * GGML_CUDA_RESTRICT dst = dst_ptr;++ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int qi = ggml_cuda_type_traits<type>::qi;+ constexpr int vdr = get_vdr_mmvq(type);+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();++ constexpr vec_dot_q_cuda_t vec_dot_q_cuda = get_vec_dot_q_cuda(type);++ const bool shared_expert = has_fusion && fusion.shared_up && blockIdx.y == gridDim.y - 1;+ if (shared_expert) {+ vx = fusion.shared_up;+ if constexpr (std::is_same_v<dst_t, float>) {+ dst = fusion.shared_dst;+ }+ stride_col_dst = fusion.shared_stride_col_dst;+ }++ // fuse gate, bias, scales, and glu_op into the up projection+ bool use_gate = false;+ const void * vgate = nullptr;+ const float * x_bias = nullptr;+ const float * gate_bias = nullptr;+ const float * x_scale = nullptr;+ const float * gate_scale = nullptr;+ ggml_glu_op active_glu = GGML_GLU_OP_SWIGLU;+ float glu_limit = 0.0f;++ if constexpr (has_fusion) {+ use_gate = fusion.gate != nullptr;+ vgate = shared_expert ? fusion.shared_gate : fusion.gate;+ x_bias = (const float *) fusion.x_bias;+ gate_bias = (const float *) fusion.gate_bias;+ active_glu = fusion.glu_op;+ glu_limit = fusion.glu_limit;+ if constexpr (type == GGML_TYPE_NVFP4) {+ x_scale = (const float *) fusion.x_scale;+ gate_scale = (const float *) fusion.gate_scale;+ }+ }++ const uint32_t token_idx = threadIdx.y;+ const int row0 = c_rows_per_block*blockIdx.x;+ const int blocks_per_row_x = ncols_x / qk;+ constexpr int blocks_per_iter = vdr * warp_size / qi;++ const uint32_t channel_dst = shared_expert ? 0 : blockIdx.y;++ if (token_idx >= ncols_dst) {+ return;+ }++ ggml_cuda_pdl_sync();+ const uint32_t channel_x = shared_expert ? 0 : ids[channel_dst + token_idx * ids_stride];+ const uint32_t channel_y = fastmodulo(channel_dst, nchannels_y);++ const block_q8_1 * y = ((const block_q8_1 *) vy) + channel_y*stride_channel_y + token_idx*stride_col_y;+ const int kbx_offset = channel_x*stride_channel_x + row0*stride_row_x;++ // partial sum for each thread+ float tmp[c_rows_per_block] = {0.0f};+ float tmp_gate[c_rows_per_block] = {0.0f};++ for (int kbx = threadIdx.x / (qi/vdr); kbx < blocks_per_row_x; kbx += blocks_per_iter) {+ const int kby = kbx * (qk/QK8_1);+ const int kqs = vdr * (threadIdx.x % (qi/vdr));++#pragma unroll+ for (int i = 0; i < c_rows_per_block; ++i) {+ tmp[i] += vec_dot_q_cuda(vx, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs);+ if constexpr (has_fusion) {+ if (use_gate) {+ tmp_gate[i] += vec_dot_q_cuda(vgate, &y[kby], kbx_offset + i*stride_row_x + kbx, kqs);+ }+ }+ }+ }++ ggml_cuda_pdl_lc();++ // Warp-level reduction only - no shared memory needed+#pragma unroll+ for (int i = 0; i < c_rows_per_block; ++i) {+ tmp[i] = warp_reduce_sum<warp_size>(tmp[i]);+ if constexpr (has_fusion) {+ if (use_gate) {+ tmp_gate[i] = warp_reduce_sum<warp_size>(tmp_gate[i]);+ }+ }+ }++ // Write results+ if (threadIdx.x < c_rows_per_block && (c_rows_per_block == 1 || uint32_t(row0 + threadIdx.x) < nrows_x)) {+ float result = tmp[threadIdx.x];+ if constexpr (has_fusion) {+ const uint32_t bias_idx = channel_x*stride_channel_dst + row0 + threadIdx.x;++ if constexpr (type == GGML_TYPE_NVFP4) {+ if (x_scale) {+ result *= x_scale[channel_x];+ }+ }+ if (x_bias) {+ result += x_bias[bias_idx];+ }+ if (use_gate) {+ float gate_value = tmp_gate[threadIdx.x];+ if constexpr (type == GGML_TYPE_NVFP4) {+ if (gate_scale) {+ gate_value *= gate_scale[channel_x];+ }+ }+ if (gate_bias) {+ gate_value += gate_bias[bias_idx];+ }+ switch (active_glu) {+ case GGML_GLU_OP_SWIGLU:+ result *= ggml_cuda_op_silu_single(gate_value);+ break;+ case GGML_GLU_OP_GEGLU:+ result *= ggml_cuda_op_gelu_single(gate_value);+ break;+ case GGML_GLU_OP_SWIGLU_OAI:+ result = ggml_cuda_op_swiglu_oai_single(gate_value, result);+ break;+ case GGML_GLU_OP_SWIGLU_CLAMP:+ result = ggml_cuda_op_swiglu_clamp_single(gate_value, result, glu_limit);+ break;+ default:+ result = result * gate_value;+ break;+ }+ }+ }+ dst[channel_dst*stride_channel_dst + token_idx*stride_col_dst + row0 + threadIdx.x] = ggml_cuda_cast<dst_t>(result);+ }++ if constexpr (!has_fusion) {+ GGML_UNUSED_VARS(use_gate, tmp_gate, vgate, x_bias, gate_bias, active_glu, glu_limit, x_scale, gate_scale);+ } else if constexpr (type != GGML_TYPE_NVFP4) {+ GGML_UNUSED_VARS(x_scale, gate_scale);+ }+}++template<ggml_type type>+static std::pair<dim3, dim3> calc_launch_params(+ const int ncols_dst, const int nrows_x, const int nchannels_dst, const int nsamples_or_ntokens,+ const int warp_size, const mmvq_parameter_table_id table_id, const bool small_k = false, const bool halve_iters = false) {+ const int nwarps = calc_nwarps(type, ncols_dst, table_id, small_k, halve_iters);+ const int rpb = calc_rows_per_block(ncols_dst, table_id, small_k, nwarps);+ const int64_t nblocks = (nrows_x + rpb - 1) / rpb;+ const dim3 block_nums(nblocks, nchannels_dst, nsamples_or_ntokens);+ const dim3 block_dims(warp_size, nwarps, 1);+ return {block_nums, block_dims};+}++template<ggml_type type, int c_ncols_dst, bool small_k = false, bool halve_iters = false, typename dst_t>+static void mul_mat_vec_q_switch_fusion(+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t stride_row_x, const uint32_t stride_col_y,+ const uint32_t stride_col_dst, const uint3 channel_ratio, const uint32_t stride_channel_x,+ const uint32_t stride_channel_y, const uint32_t stride_channel_dst, const uint3 sample_ratio,+ const uint32_t stride_sample_x, const uint32_t stride_sample_y, const uint32_t stride_sample_dst,+ const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared,+ const uint32_t ids_stride, cudaStream_t stream) {++ const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||+ fusion.x_scale != nullptr || fusion.gate_scale != nullptr;+ if constexpr (c_ncols_dst == 1) {+ if (has_fusion) {+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);+ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, true, small_k, halve_iters, dst_t>, launch_params,+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);+ return;+ }+ }++ GGML_ASSERT(!has_fusion && "fusion only supported for ncols_dst=1");++ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, nbytes_shared, stream);+ ggml_cuda_kernel_launch(mul_mat_vec_q<type, c_ncols_dst, false, small_k, halve_iters, dst_t>, launch_params,+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride);+}++template <ggml_type type, typename dst_t>+static void mul_mat_vec_q_moe_launch(+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,+ const uint32_t ncols_x, const uint3 nchannels_y, const uint32_t nrows_x,+ const uint32_t stride_row_x, const uint32_t stride_col_y, const uint32_t stride_col_dst,+ const uint32_t stride_channel_x, const uint32_t stride_channel_y, const uint32_t stride_channel_dst,+ const uint32_t ncols_dst, const uint32_t ids_stride,+ const int warp_size, const int nchannels_dst, cudaStream_t stream) {++ constexpr int rows_per_block = 2; // 2 gives best perf based on tuning+ const int64_t nblocks_rows = (nrows_x + rows_per_block - 1) / rows_per_block;+ const dim3 block_nums(nblocks_rows, nchannels_dst + (fusion.shared_up != nullptr));+ const dim3 block_dims(warp_size, ncols_dst);+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(block_nums, block_dims, 0, stream);++ const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||+ fusion.x_scale != nullptr || fusion.gate_scale != nullptr;++ if (has_fusion) {+ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, true, dst_t>, launch_params,+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,+ stride_row_x, stride_col_y, stride_col_dst,+ stride_channel_x, stride_channel_y, stride_channel_dst,+ ncols_dst, ids_stride);+ } else {+ ggml_cuda_kernel_launch(mul_mat_vec_q_moe<type, rows_per_block, false, dst_t>, launch_params,+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y, nrows_x,+ stride_row_x, stride_col_y, stride_col_dst,+ stride_channel_x, stride_channel_y, stride_channel_dst,+ ncols_dst, ids_stride);+ }+}++template <ggml_type type, typename dst_t>+static void mul_mat_vec_q_switch_ncols_dst(+ const void * vx, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,+ const int ncols_x, const int nrows_x, const int ncols_dst,+ const int stride_row_x, const int stride_col_y, const int stride_col_dst,+ const int nchannels_x, const int nchannels_y, const int nchannels_dst,+ const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,+ const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,+ const int ids_stride, cudaStream_t stream) {++ GGML_ASSERT(ncols_x % ggml_blck_size(type) == 0);+ GGML_ASSERT(ncols_dst <= MMVQ_MAX_BATCH_SIZE);++ const uint3 nchannels_y_fd = ids ? init_fastdiv_values(nchannels_y) : make_uint3(0, 0, 0);+ const uint3 channel_ratio_fd = ids ? make_uint3(0, 0, 0) : init_fastdiv_values(nchannels_dst / nchannels_x);+ const uint3 sample_ratio_fd = init_fastdiv_values(nsamples_dst / nsamples_x);++ const int device = ggml_cuda_get_device();+ const int cc = ggml_cuda_info().devices[device].cc;+ const int warp_size = ggml_cuda_info().devices[device].warp_size;+ const mmvq_parameter_table_id table_id = get_device_table_id(cc);++ const bool has_ids = ids != nullptr;++ // How the K loop divides up at the baseline block width, both decisions below use these.+ constexpr int qk = ggml_cuda_type_traits<type>::qk;+ constexpr int qi = ggml_cuda_type_traits<type>::qi;+ constexpr int vdr = get_vdr_mmvq(type);+ const int blocks_per_row_x = ncols_x / qk;+ const int blocks_per_iter_1warp = vdr * warp_size / qi;++ const auto should_use_small_k = [&](int c_ncols_dst) {+ // When K is small, increase rows_per_block to match nwarps so each warp has more work to do+ // Trigger when the full thread block covers all K blocks in a single loop iteration and few threads remain idle.+ const int nwarps = calc_nwarps(type, c_ncols_dst, table_id);+ bool use = nwarps > 1 && blocks_per_row_x < nwarps * blocks_per_iter_1warp;++ constexpr std::array<ggml_type, 2> iq_slow_turing = {+ GGML_TYPE_IQ3_XXS,+ GGML_TYPE_IQ3_S,+ };+ constexpr std::array<ggml_type, 8> iq_slow_other = {+ GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS,+ GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS,+ };+ constexpr std::array<ggml_type, 3> slow_pascal = {+ GGML_TYPE_IQ3_S,+ GGML_TYPE_Q2_K,+ GGML_TYPE_Q3_K,+ };++ const bool is_nvidia_turing_plus = GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_TURING;+ const bool is_nvidia_pascal_older = GGML_CUDA_CC_IS_NVIDIA(cc) && cc < GGML_CUDA_CC_VOLTA;++ if (is_nvidia_turing_plus) {+ if (ncols_dst == 1 &&+ std::find(iq_slow_turing.begin(), iq_slow_turing.end(), type) != iq_slow_turing.end()) {+ use = false;+ }+ } else if ((ncols_dst == 1 && std::find(iq_slow_other.begin(), iq_slow_other.end(), type) != iq_slow_other.end()) ||+ (is_nvidia_pascal_older && std::find(slow_pascal.begin(), slow_pascal.end(), type) != slow_pascal.end()) ||+ GGML_CUDA_CC_IS_RDNA(cc)) {+ use = false;+ }++ return use;+ };++ // Whether doubling nwarps pays off on the ncols_dst == 1 path, where K sets the K loop trip count.+ const auto should_halve_iters = [&] {+ if (table_id != MMVQ_PARAMETERS_GB10) {+ return false;+ }++ // Expert rows are gathered per token, so a wider block adds reduction work without reuse.+ if (has_ids) {+ return false;+ }++ const int blocks_per_iter = calc_nwarps(type, 1, table_id) * blocks_per_iter_1warp;+ const int iters = (blocks_per_row_x + blocks_per_iter - 1) / blocks_per_iter;+ const int iters_wide = (blocks_per_row_x + blocks_per_iter * 2 - 1) / (blocks_per_iter * 2);++ // An odd trip count leaves half the wider block idle for its last iteration, that tail is+ // only affordable once the loop is long enough to dilute it to an eighth of the work (observation).+ const int idle = iters_wide * 2 - iters;++ return idle * 8 <= iters_wide * 2;+ };++ if (has_ids && ncols_dst > 1) {+ // Multi-token MUL_MAT_ID path - dedicated MoE kernel+ mul_mat_vec_q_moe_launch<type>(+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, nrows_x,+ stride_row_x, stride_col_y, stride_col_dst,+ stride_channel_x, stride_channel_y, stride_channel_dst,+ ncols_dst, ids_stride, warp_size, nchannels_dst, stream);+ return;+ }++ switch (ncols_dst) {+ case 1: {+ // static, else MSVC lambda capture breaks the constexpr uses below+ static constexpr int c_ncols_dst = 1;++ // Tag types keep the flags compile-time, so __launch_bounds__ matches what is launched.+ const auto launch = [&](auto small_k_tag, auto halve_iters_tag) {+ constexpr bool c_small_k = decltype(small_k_tag)::value;+ // Types the table does not promote would compile a second, identical kernel.+ constexpr bool c_promoted =+ calc_nwarps(type, c_ncols_dst, MMVQ_PARAMETERS_GB10, false, true) !=+ calc_nwarps(type, c_ncols_dst, MMVQ_PARAMETERS_GB10, false, false);++ constexpr bool c_halve_iters = decltype(halve_iters_tag)::value && c_promoted;++ const std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst + (fusion.shared_up != nullptr),+ nsamples_dst, warp_size, table_id, c_small_k, c_halve_iters);+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst, c_small_k, c_halve_iters>(+ vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio_fd,+ stride_sample_x, stride_sample_y, stride_sample_dst, dims.first, dims.second, 0, ids_stride,+ stream);+ };++ if (should_use_small_k(c_ncols_dst)) {+ launch(std::true_type{}, std::false_type{});+ } else if (should_halve_iters()) {+ launch(std::false_type{}, std::true_type{});+ } else {+ launch(std::false_type{}, std::false_type{});+ }+ } break;+ case 2: {+ constexpr int c_ncols_dst = 2;+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,+ dims.first, dims.second, 0, ids_stride, stream);+ } break;+ case 3: {+ constexpr int c_ncols_dst = 3;+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,+ dims.first, dims.second, 0, ids_stride, stream);+ } break;+ case 4: {+ constexpr int c_ncols_dst = 4;+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,+ dims.first, dims.second, 0, ids_stride, stream);+ } break;+ case 5: {+ constexpr int c_ncols_dst = 5;+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,+ dims.first, dims.second, 0, ids_stride, stream);+ } break;+ case 6: {+ constexpr int c_ncols_dst = 6;+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,+ dims.first, dims.second, 0, ids_stride, stream);+ } break;+ case 7: {+ constexpr int c_ncols_dst = 7;+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,+ dims.first, dims.second, 0, ids_stride, stream);+ } break;+ case 8: {+ constexpr int c_ncols_dst = 8;+ std::pair<dim3, dim3> dims = calc_launch_params<type>(c_ncols_dst, nrows_x, nchannels_dst, nsamples_dst, warp_size, table_id);+ mul_mat_vec_q_switch_fusion<type, c_ncols_dst>(vx, vy, ids, fusion, dst, ncols_x, nchannels_y_fd, stride_row_x, stride_col_y, stride_col_dst,+ channel_ratio_fd, stride_channel_x, stride_channel_y, stride_channel_dst,+ sample_ratio_fd, stride_sample_x, stride_sample_y, stride_sample_dst,+ dims.first, dims.second, 0, ids_stride, stream);+ } break;+ default:+ GGML_ABORT("fatal error");+ }+}++template <typename dst_t>+static void mul_mat_vec_q_switch_type(+ const void * vx, const ggml_type type_x, const void * vy, const int32_t * ids, const ggml_cuda_mm_fusion_args_device fusion, dst_t * dst,+ const int ncols_x, const int nrows_x, const int ncols_dst,+ const int stride_row_x, const int stride_col_y, const int stride_col_dst,+ const int nchannels_x, const int nchannels_y, const int nchannels_dst,+ const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,+ const int nsamples_x, const int nsamples_dst, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst,+ const int ids_stride, cudaStream_t stream) {+ switch (type_x) {+ case GGML_TYPE_Q1_0:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q1_0>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q2_0:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q2_0>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q4_0:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_0>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q4_1:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_1>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q5_0:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_0>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q5_1:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_1>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q8_0:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q8_0>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_MXFP4:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_MXFP4>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_NVFP4:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_NVFP4>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q2_K:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q2_K>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q3_K:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q3_K>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q4_K:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q4_K>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q5_K:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q5_K>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_Q6_K:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_Q6_K>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ2_XXS:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_XXS>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ2_XS:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_XS>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ2_S:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ2_S>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ3_XXS:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ3_XXS>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ1_S:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ1_S>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ1_M:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ1_M>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ4_NL:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ4_NL>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ4_XS:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ4_XS>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ case GGML_TYPE_IQ3_S:+ mul_mat_vec_q_switch_ncols_dst<GGML_TYPE_IQ3_S>+ (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst,+ nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,+ nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream);+ break;+ default:+ GGML_ABORT("fatal error");+ }+}++void ggml_cuda_mul_mat_vec_q(+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst,+ const ggml_cuda_mm_fusion_args_host * fusion, const void * src1_q8_1_pre) {+ // FreeToken: activations and results may also be bf16 or fp16+ GGML_ASSERT( src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_BF16 || src1->type == GGML_TYPE_F16);+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_BF16 || dst->type == GGML_TYPE_F16);+ GGML_ASSERT(!ids || ids->type == GGML_TYPE_I32); // Optional, used for batched GGML_MUL_MAT_ID.++ GGML_TENSOR_BINARY_OP_LOCALS;++ cudaStream_t stream = ctx.stream();++ const size_t ts_src0 = ggml_type_size(src0->type);+ const size_t ts_src1 = ggml_type_size(src1->type);+ const size_t ts_dst = ggml_type_size(dst->type);++ GGML_ASSERT( nb00 == ts_src0);+ GGML_ASSERT( nb10 == ts_src1);+ GGML_ASSERT( nb0 == ts_dst);+ GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));++ GGML_ASSERT(!ids || ne12 <= MMVQ_MAX_BATCH_SIZE);++ const float * src1_d = (const float *) src1->data;+ const int32_t * ids_d = ids ? (const int32_t *) ids->data : nullptr;+ float * dst_d = (float *) dst->data;++ ggml_cuda_mm_fusion_args_device fusion_local{};++ if (fusion) {+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;+ GGML_ASSERT( !ids || dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc));+ GGML_ASSERT( ids || dst->ne[1] == 1);+ // Scale fusion is only allowed for NVFP4 currently as the cost of checking this at run-time in the prologue is+ // non-negligible for some models such as gpt-oss-20b+ GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_NVFP4);++ if (fusion->shared_up) {+ GGML_ASSERT(ids && fusion->gate && fusion->shared_gate && fusion->shared_dst);+ GGML_ASSERT(!fusion->x_bias && !fusion->gate_bias && !fusion->x_scale && !fusion->gate_scale);+ GGML_ASSERT(ne11 == 1 && ne03 == 1 && ne13 == 1);+ GGML_ASSERT(fusion->shared_up->type == src0->type && fusion->shared_gate->type == src0->type);+ GGML_ASSERT(ggml_are_same_shape(fusion->shared_up, fusion->shared_gate));+ GGML_ASSERT(ggml_is_contiguous(fusion->shared_up) && ggml_is_contiguous(fusion->shared_gate));+ GGML_ASSERT(fusion->shared_up->ne[0] == ne00 && fusion->shared_up->ne[1] == ne01);+ GGML_ASSERT(fusion->shared_up->nb[1] == nb01 && ggml_is_matrix(fusion->shared_up));+ GGML_ASSERT(fusion->shared_dst->type == GGML_TYPE_F32 && ggml_is_contiguous(fusion->shared_dst));+ GGML_ASSERT(fusion->shared_dst->ne[0] == ne0 && fusion->shared_dst->ne[1] == ne2);+ fusion_local.shared_up = fusion->shared_up->data;+ fusion_local.shared_gate = fusion->shared_gate->data;+ fusion_local.shared_dst = (float *) fusion->shared_dst->data;+ fusion_local.shared_stride_col_dst = fusion->shared_dst->nb[1] / ts_dst;+ }++ if (fusion->x_bias) {+ GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32);+ GGML_ASSERT(fusion->x_bias->ne[0] == dst->ne[0]);+ GGML_ASSERT(!ids || fusion->x_bias->ne[1] == src0->ne[2]);+ fusion_local.x_bias = fusion->x_bias->data;+ }+ if (fusion->gate) {+ GGML_ASSERT(fusion->gate->type == src0->type && ggml_are_same_stride(fusion->gate, src0));+ fusion_local.gate = fusion->gate->data;+ }+ if (fusion->gate_bias) {+ GGML_ASSERT(fusion->gate_bias->type == GGML_TYPE_F32);+ GGML_ASSERT(fusion->gate_bias->ne[0] == dst->ne[0]);+ GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);+ fusion_local.gate_bias = fusion->gate_bias->data;+ }+ if (fusion->x_scale) {+ GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32);+ GGML_ASSERT(ggml_is_contiguous(fusion->x_scale));+ GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1));+ fusion_local.x_scale = fusion->x_scale->data;+ }+ if (fusion->gate_scale) {+ GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32);+ GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale));+ GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));+ fusion_local.gate_scale = fusion->gate_scale->data;+ }+ fusion_local.glu_op = fusion->glu_op;+ fusion_local.glu_limit = fusion->glu_limit;+ }++ // If src0 is a temporary compute buffer, clear any potential padding.+ if (ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {+ const size_t size_data = ggml_nbytes(src0);+ const size_t size_alloc = ggml_backend_buffer_get_alloc_size(src0->buffer, src0);+ if (size_alloc > size_data) {+ GGML_ASSERT(ggml_is_contiguously_allocated(src0));+ GGML_ASSERT(!src0->view_src);+ CUDA_CHECK(cudaMemsetAsync((char *) src0->data + size_data, 0, size_alloc - size_data, stream));+ }+ }++ const int64_t ne10_padded = GGML_PAD(ne10, MATRIX_ROW_PADDING);+ // FreeToken: src1_q8_1_pre is src1 already quantized by quantize_row_q8_1_cuda with this padding and layout+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool());+ const void * src1_q8 = src1_q8_1_pre;+ if (src1_q8 == nullptr) {+ src1_q8_1.alloc(ne13*ne12 * ne11*ne10_padded * sizeof(block_q8_1)/QK8_1);+ src1_q8 = src1_q8_1.get();+ const int64_t s11 = src1->nb[1] / ts_src1;+ const int64_t s12 = src1->nb[2] / ts_src1;+ const int64_t s13 = src1->nb[3] / ts_src1;+ if (src1->type == GGML_TYPE_BF16) {+ quantize_row_q8_1_cuda((const nv_bfloat16 *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);+ } else if (src1->type == GGML_TYPE_F16) {+ quantize_row_q8_1_cuda((const half *) src1->data, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);+ } else {+ quantize_row_q8_1_cuda(src1_d, nullptr, src1_q8_1.get(), src0->type, ne10, s11, s12, s13, ne10_padded, ne11, ne12, ne13, stream);+ }+ }++ const int64_t s01 = src0->nb[1] / ts_src0;+ const int64_t s11 = ne10_padded / QK8_1;+ const int64_t s1 = dst->nb[1] / ts_dst;+ const int64_t s02 = src0->nb[2] / ts_src0;+ const int64_t s2 = dst->nb[2] / ts_dst;+ const int64_t s03 = src0->nb[3] / ts_src0;+ const int64_t s3 = dst->nb[3] / ts_dst;++ const int64_t s12 = ne11*s11;+ const int64_t s13 = ne12*s12;++ // For MUL_MAT_ID the memory layout is different than for MUL_MAT:+ const int64_t ncols_dst = ids ? ne2 : ne1;+ const int64_t nchannels_y = ids ? ne11 : ne12;+ const int64_t nchannels_dst = ids ? ne1 : ne2;+ const int64_t stride_col_dst = ids ? s2 : s1;+ const int64_t stride_col_y = ids ? s12 : s11;+ const int64_t stride_channel_dst = ids ? s1 : s2;+ const int64_t stride_channel_y = ids ? s11 : s12;++ const int64_t ids_stride = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;++ const auto launch = [&](auto * dst_ptr) {+ mul_mat_vec_q_switch_type(+ src0->data, src0->type, src1_q8, ids_d, fusion_local, dst_ptr, ne00,+ ne01, ncols_dst, s01, stride_col_y, stride_col_dst,+ ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,+ ne03, ne3, s03, s13, s3, ids_stride, stream);+ };+ if (dst->type == GGML_TYPE_BF16) {+ // FreeToken: the fused shared expert writes an fp32 shared_dst+ GGML_ASSERT(!fusion || !fusion->shared_up);+ launch((nv_bfloat16 *) dst->data);+ } else if (dst->type == GGML_TYPE_F16) {+ GGML_ASSERT(!fusion || !fusion->shared_up);+ launch((half *) dst->data);+ } else {+ launch(dst_d);+ }+}++void ggml_cuda_op_mul_mat_vec_q(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream) {++ const int64_t ne00 = src0->ne[0];+ const int64_t row_diff = row_high - row_low;++ const int64_t ne10 = src1->ne[0];+ GGML_ASSERT(ne10 % QK8_1 == 0);++ const int64_t ne0 = dst->ne[0];++ int id = ggml_cuda_get_device();++ // the main device has a larger memory buffer to hold the results from all GPUs+ // nrows_dst == nrows of the matrix that the kernel writes into+ const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff;++ const int stride_row_x = ne00 / ggml_blck_size(src0->type);+ const int stride_col_y = src1_padded_row_size / QK8_1;++ ggml_cuda_mm_fusion_args_device fusion_local{};+ mul_mat_vec_q_switch_type(+ src0_dd_i, src0->type, src1_ddq_i, nullptr, fusion_local, dst_dd_i, ne00, row_diff, src1_ncols, stride_row_x, stride_col_y, nrows_dst,+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, stream);++ GGML_UNUSED_VARS(src1, dst, src1_ddf_i, src1_ncols, src1_padded_row_size);+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cuhnew file mode 100644index 00000000..6973780c--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/mmvq.cuh@@ -0,0 +1,20 @@+#include "common.cuh"++#define MMVQ_MAX_BATCH_SIZE 8 // Max. batch size for which to use MMVQ kernels.++bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11);++// Returns the maximum batch size for which MMVQ should be used for MUL_MAT_ID,+// based on the quantization type and GPU architecture (compute capability).+int get_mmvq_mmid_max_batch(ggml_type type, int cc);++// FreeToken: src1_q8_1_pre skips the internal quantization of src1+void ggml_cuda_mul_mat_vec_q(ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst, const ggml_cuda_mm_fusion_args_host * fusion = nullptr,+ const void * src1_q8_1_pre = nullptr);++void ggml_cuda_op_mul_mat_vec_q(+ ggml_backend_cuda_context & ctx,+ const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,+ const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,+ const int64_t src1_padded_row_size, cudaStream_t stream);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cunew file mode 100644index 00000000..46c9f861--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cu@@ -0,0 +1,729 @@+#include "quantize.cuh"+#include "convert.cuh"+#include <cstdint>++#if defined(BLACKWELL_MMA_AVAILABLE)+// this maps to 256-bit loads in PTX on supported devices,+// and otherwise falls back to 2 128-bit loads+struct __builtin_align__(32) float8 {+ float x; float y; float z; float w;+ float p; float q; float r; float s;+};++#if CUDART_VERSION >= 12080+static __device__ __forceinline__ float nvfp4_native_scale_error(+ const float vals[QK_NVFP4_SUB], const float inv_col_scale, const float inv_scale, const float scale) {+ const float scale_dequant = 2.0f * scale;+ float err = 0.0f;++#pragma unroll+ for (int k = 0; k < QK_NVFP4_SUB; k += 4) {+ const float v0 = vals[k + 0] * inv_col_scale;+ const float v1 = vals[k + 1] * inv_col_scale;+ const float v2 = vals[k + 2] * inv_col_scale;+ const float v3 = vals[k + 3] * inv_col_scale;++ const __nv_fp4x4_e2m1 q(make_float4(v0 * inv_scale, v1 * inv_scale, v2 * inv_scale, v3 * inv_scale));+ const __nv_fp4x4_storage_t q_storage = q.__x;+ const __nv_fp4x2_storage_t q_lo = static_cast<__nv_fp4x2_storage_t>(q_storage);+ const __nv_fp4x2_storage_t q_hi = static_cast<__nv_fp4x2_storage_t>(q_storage >> 8U);++ const __half2_raw hraw2_lo = __nv_cvt_fp4x2_to_halfraw2(q_lo, __NV_E2M1);+ const __half2_raw hraw2_hi = __nv_cvt_fp4x2_to_halfraw2(q_hi, __NV_E2M1);+ const __half2 h2_lo = static_cast<__half2>(hraw2_lo);+ const __half2 h2_hi = static_cast<__half2>(hraw2_hi);+ const float2 dq_lo = __half22float2(h2_lo);+ const float2 dq_hi = __half22float2(h2_hi);++ const float err0 = fabsf(v0) - fabsf(dq_lo.x) * scale_dequant;+ const float err1 = fabsf(v1) - fabsf(dq_lo.y) * scale_dequant;+ const float err2 = fabsf(v2) - fabsf(dq_hi.x) * scale_dequant;+ const float err3 = fabsf(v3) - fabsf(dq_hi.y) * scale_dequant;++ err = fmaf(err0, err0, err);+ err = fmaf(err1, err1, err);+ err = fmaf(err2, err2, err);+ err = fmaf(err3, err3, err);+ }++ return err;+}+#endif // CUDART_VERSION >= 12080+#endif // defined(BLACKWELL_MMA_AVAILABLE)++template <typename src_t>+__launch_bounds__(CUDA_QUANTIZE_BLOCK_SIZE, 1)+static __global__ void quantize_q8_1(+ const src_t * x_ptr, void * vy_ptr,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const uint32_t ne1, const uint3 ne2) {+ ggml_cuda_pdl_lc();+ const src_t * GGML_CUDA_RESTRICT x = x_ptr;+ void * GGML_CUDA_RESTRICT vy = vy_ptr;+ const int64_t i0 = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;++ if (i0 >= ne0) {+ return;+ }++ const int64_t i3 = fastdiv(blockIdx.z, ne2);+ const int64_t i2 = blockIdx.z - i3*ne2.z;+ const int64_t i1 = blockIdx.y;++ const int64_t & i00 = i0;+ const int64_t & i01 = i1;+ const int64_t & i02 = i2;+ const int64_t & i03 = i3;++ const int64_t i_cont = ((i3*ne2.z + i2) * ne1 + i1) * ne0 + i0;++ block_q8_1 * y = (block_q8_1 *) vy;++ const int64_t ib = i_cont / QK8_1; // block index+ const int64_t iqs = i_cont % QK8_1; // quant index++ ggml_cuda_pdl_sync();+ const float xi = i0 < ne00 ? ggml_cuda_cast<float>(x[i03*s03 + i02*s02 + i01*s01 + i00]) : 0.0f;+ float amax = fabsf(xi);+ float sum = xi;++ amax = warp_reduce_max<QK8_1>(amax);+ sum = warp_reduce_sum<QK8_1>(sum);++ const float d = amax / 127.0f;+ const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);++ y[ib].qs[iqs] = q;++ if (iqs > 0) {+ return;+ }++ y[ib].ds = make_half2(d, sum);+}++__device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) {+ if (!(amax > 0.0f)) {+ return 0;+ }++ // FP4 E2M1: max exponent (unbiased) is 2.+ constexpr int FP4_E2M1_EMAX = 2;++ const float e = log2f(amax);++ // "even" -> round-to-nearest integer, ties-to-even+ const int e_int = __float2int_rn(e);++ const int shared_exp = e_int - FP4_E2M1_EMAX;++ int biased = shared_exp + 127;++ biased = max(biased, 0);+ biased = min(biased, 254);++ return static_cast<uint8_t>(biased);+}++// scatter: grid over tokens, quantize once, write to all the token's compact rows+template <bool scatter, bool use_aligned_float8>+static __global__ void quantize_mmq_nvfp4(+ const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, float * __restrict__ scale,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int n_expert_used) {+#if defined(BLACKWELL_MMA_AVAILABLE)++ int64_t base_idx;+ if constexpr (scatter) {+ base_idx = (int64_t) blockIdx.x * s02; // one physical row per token+ } else {+ const int64_t i2 = blockIdx.y % ne2;+ const int64_t i3 = blockIdx.y / ne2;+ const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x;+ base_idx = i3 * s03 + i2 * s02 + i01 * s01;+ }+ const float * __restrict__ x_row = x + base_idx;++ float amax = 0.0f;+ if constexpr (use_aligned_float8) {+ for (int64_t i0 = 8 * threadIdx.x; i0 < ne00; i0 += 8 * blockDim.x) {+ const float * x_base = x_row + i0;+ const float8 v = reinterpret_cast<const float8 *>(x_base)[0];+ amax = fmaxf(amax, fabsf(v.x));+ amax = fmaxf(amax, fabsf(v.y));+ amax = fmaxf(amax, fabsf(v.z));+ amax = fmaxf(amax, fabsf(v.w));+ amax = fmaxf(amax, fabsf(v.p));+ amax = fmaxf(amax, fabsf(v.q));+ amax = fmaxf(amax, fabsf(v.r));+ amax = fmaxf(amax, fabsf(v.s));+ }+ } else {+ for (int64_t i0 = threadIdx.x; i0 < ne00; i0 += blockDim.x) {+ amax = fmaxf(amax, fabsf(x_row[i0]));+ }+ }++ amax = warp_reduce_max<WARP_SIZE>(amax);++ __shared__ float warp_amax[CUDA_QUANTIZE_BLOCK_SIZE_MMQ / WARP_SIZE];+ const int lane = threadIdx.x % WARP_SIZE;+ const int warp = threadIdx.x / WARP_SIZE;++ if (lane == 0) {+ warp_amax[warp] = amax;+ }+ __syncthreads();++ if (warp == 0) {+ amax = threadIdx.x < int(CUDA_QUANTIZE_BLOCK_SIZE_MMQ / WARP_SIZE) ? warp_amax[lane] : 0.0f;+ amax = warp_reduce_max<WARP_SIZE>(amax);+ if (lane == 0) {+ warp_amax[0] = amax / (6.0f * 448.0f);+ if constexpr (scatter) {+#pragma unroll+ for (int slot = 0; slot < n_expert_used; ++slot) {+ const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];+ scale[i] = warp_amax[0];+ }+ } else {+ scale[blockIdx.y * ne1 + blockIdx.x] = warp_amax[0];+ }+ }+ }+ __syncthreads();++ block_fp4_mmq * y = (block_fp4_mmq *) vy;+ const int64_t n_subblocks = (ne0 + QK_NVFP4_SUB - 1) / QK_NVFP4_SUB;++ for (int64_t isb = threadIdx.x; isb < n_subblocks; isb += blockDim.x) {+ const int64_t i0_base = isb * QK_NVFP4_SUB;+ const int64_t k_block = i0_base / QK_FP4_MMQ;+ const int sub = (i0_base % QK_FP4_MMQ) / QK_NVFP4_SUB;++ const float row_scale = warp_amax[0];+ const float inv_col_scale = row_scale > 0.0f ? 1.0f / row_scale : 0.0f;++ float vals[QK_NVFP4_SUB];+ if constexpr (use_aligned_float8) {+ const float * x_base = x_row + i0_base;+ const float8 v0 = i0_base + 7 < ne00 ? reinterpret_cast<const float8 *>(x_base)[0] : float8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};+ const float8 v1 = i0_base + 15 < ne00 ? reinterpret_cast<const float8 *>(x_base + 8)[0] : float8{0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};+ vals[0] = v0.x; vals[1] = v0.y; vals[2] = v0.z; vals[3] = v0.w;+ vals[4] = v0.p; vals[5] = v0.q; vals[6] = v0.r; vals[7] = v0.s;+ vals[8] = v1.x; vals[9] = v1.y; vals[10] = v1.z; vals[11] = v1.w;+ vals[12] = v1.p; vals[13] = v1.q; vals[14] = v1.r; vals[15] = v1.s;+ } else {+#pragma unroll+ for (int k = 0; k < QK_NVFP4_SUB; ++k) {+ const int64_t i00 = i0_base + k;+ vals[k] = i00 < ne00 ? x_row[i00] : 0.0f;+ }+ }++ uint32_t q0 = 0;+ uint32_t q1 = 0;++ float amax_sub = 0.0f;+#pragma unroll+ for (int k = 0; k < QK_NVFP4_SUB; ++k) {+ amax_sub = fmaxf(amax_sub, fabsf(vals[k] * inv_col_scale));+ }++ static constexpr int test_offsets[5] = { 0, -1, 1, -2, 2 };+ const int first_fp8_code = (int) ggml_cuda_fp32_to_ue4m3(amax_sub / 6.0f);++ uint8_t fp8_code = (uint8_t) first_fp8_code;+ float subblock_scale = ggml_cuda_ue4m3_to_fp32(fp8_code);+ float inv_scale_err = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f;+#if CUDART_VERSION >= 12080+ float best_err = nvfp4_native_scale_error(vals, inv_col_scale, inv_scale_err, subblock_scale);+#else+ float best_err = 0.0f;+#pragma unroll+ for (int k = 0; k < QK_NVFP4_SUB; ++k) {+ const float v = vals[k] * inv_col_scale;+ const uint8_t q = ggml_cuda_float_to_fp4_e2m1(v, inv_scale_err);+ const float err_diff = fabsf(v) - fabsf(kvalues_fp4[q & 0x7]) * subblock_scale;+ best_err = fmaf(err_diff, err_diff, best_err);+ }+#endif // CUDART_VERSION >= 12080++#pragma unroll+ for (int i = 1; i < 5; ++i) {+ const int test_code = first_fp8_code + test_offsets[i];+ if (test_code < 0 || test_code > 0x7e) {+ continue;+ }++ const float test_scale = ggml_cuda_ue4m3_to_fp32((uint8_t) test_code);+ const float test_inv_scale = test_scale > 0.0f ? 0.5f / test_scale : 0.0f;+#if CUDART_VERSION >= 12080+ const float cur_err = nvfp4_native_scale_error(vals, inv_col_scale, test_inv_scale, test_scale);+#else+ float cur_err = 0.0f;+#pragma unroll+ for (int k = 0; k < QK_NVFP4_SUB; ++k) {+ const float v = vals[k] * inv_col_scale;+ const uint8_t q = ggml_cuda_float_to_fp4_e2m1(v, test_inv_scale);+ const float err_diff = fabsf(v) - fabsf(kvalues_fp4[q & 0x7]) * test_scale;+ cur_err = fmaf(err_diff, err_diff, cur_err);+ }+#endif // CUDART_VERSION >= 12080++ if (cur_err < best_err) {+ best_err = cur_err;+ fp8_code = (uint8_t) test_code;+ subblock_scale = test_scale;+ }+ }+#if CUDART_VERSION >= 12080+ const float inv_scale = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f;+ const float s = inv_col_scale * inv_scale;++ __nv_fp4x4_e2m1 q0_lo(make_float4(vals[0] * s, vals[8] * s, vals[1] * s, vals[9] * s));+ __nv_fp4x4_e2m1 q0_hi(make_float4(vals[2] * s, vals[10] * s, vals[3] * s, vals[11] * s));+ __nv_fp4x4_e2m1 q1_lo(make_float4(vals[4] * s, vals[12] * s, vals[5] * s, vals[13] * s));+ __nv_fp4x4_e2m1 q1_hi(make_float4(vals[6] * s, vals[14] * s, vals[7] * s, vals[15] * s));++ const char2 q0_lo_c = *reinterpret_cast<char2 *>(&q0_lo);+ const char2 q0_hi_c = *reinterpret_cast<char2 *>(&q0_hi);+ const char2 q1_lo_c = *reinterpret_cast<char2 *>(&q1_lo);+ const char2 q1_hi_c = *reinterpret_cast<char2 *>(&q1_hi);++ q0 = uint32_t(uint8_t(q0_lo_c.x)) | (uint32_t(uint8_t(q0_lo_c.y)) << 8) |+ (uint32_t(uint8_t(q0_hi_c.x)) << 16) | (uint32_t(uint8_t(q0_hi_c.y)) << 24);+ q1 = uint32_t(uint8_t(q1_lo_c.x)) | (uint32_t(uint8_t(q1_lo_c.y)) << 8) |+ (uint32_t(uint8_t(q1_hi_c.x)) << 16) | (uint32_t(uint8_t(q1_hi_c.y)) << 24);+#else+ const float inv_scale = subblock_scale > 0.0f ? 0.5f / subblock_scale : 0.0f;+#pragma unroll+ for (int k = 0; k < QK_NVFP4_SUB / 4; ++k) {+ q0 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 0] * inv_col_scale, inv_scale)) << (8 * k);+ q0 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 8] * inv_col_scale, inv_scale)) << (8 * k + 4);+ q1 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 4] * inv_col_scale, inv_scale)) << (8 * k);+ q1 |= uint32_t(ggml_cuda_float_to_fp4_e2m1(vals[k + 12] * inv_col_scale, inv_scale)) << (8 * k + 4);+ }+#endif // CUDART_VERSION >= 12080++ if constexpr (scatter) {+#pragma unroll+ for (int slot = 0; slot < n_expert_used; ++slot) {+ const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];+ block_fp4_mmq * yb = y + (k_block * ne1 + i);+ uint32_t * yqs = reinterpret_cast<uint32_t *>(yb->qs);+ yqs[2 * sub + 0] = q0;+ yqs[2 * sub + 1] = q1;+ reinterpret_cast<uint8_t *>(yb->d4)[sub] = fp8_code;+ }+ } else {+ const int64_t blocks_per_col = (ne0 + QK_FP4_MMQ - 1) / QK_FP4_MMQ;+ block_fp4_mmq * yb = y + (blockIdx.y * (blocks_per_col * ne1) + k_block * ne1 + blockIdx.x);+ uint32_t * yqs = reinterpret_cast<uint32_t *>(yb->qs);+ yqs[2 * sub + 0] = q0;+ yqs[2 * sub + 1] = q1;+ reinterpret_cast<uint8_t *>(yb->d4)[sub] = fp8_code;+ }+ }+#else+ GGML_UNUSED_VARS(x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, n_expert_used);+ NO_DEVICE_CODE; // This is for Blackwell NVFP4 activations only.+#endif // defined(BLACKWELL_MMA_AVAILABLE)++}++// quantize values in the format mxfp4 is stored which is interleaved nibbles+// i.e. a block a0-a31 is represented as a0a16,a1a17 ...a15a31+// scatter: grid over tokens, quantize once, write to all the token's compact rows+template <bool scatter>+static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x,+ const int32_t * __restrict__ ids,+ void * __restrict__ vy,+ const int64_t ne00,+ const int64_t s01,+ const int64_t s02,+ const int64_t s03,+ const int64_t ne0,+ const int ne1,+ const int ne2,+ const int n_expert_used) {+ constexpr int vals_per_scale = 32;+ constexpr int vals_per_warp = 2 * vals_per_scale; // Each warp processes 2 blocks of 32 = 64 values++ const int warp_id = threadIdx.y;+ const int lane_id_32 = threadIdx.x;++ const int nwarps = blockDim.y;++ const int64_t warp_start_offset = (blockIdx.y * nwarps + warp_id) * vals_per_warp;++ if (warp_start_offset >= ne0) {+ return;+ }++ const int64_t block_fp4_mmq_size = QK_FP4_MMQ;+ const int64_t k_block = warp_start_offset / block_fp4_mmq_size;+ const int64_t quad_idx_in_block = (warp_start_offset % block_fp4_mmq_size) / vals_per_warp;++ const int group_id = lane_id_32 / 4;+ const int lane_in_group = lane_id_32 % 4;+ const int base = group_id * 2;++ ggml_cuda_pdl_sync();+ int64_t base_pos;+ if constexpr (scatter) {+ base_pos = (int64_t) blockIdx.x * s02; // one physical row per token+ } else {+ const int64_t i2 = blockIdx.z % ne2;+ const int64_t i3 = blockIdx.z / ne2;+ const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x;+ base_pos = i3 * s03 + i2 * s02 + i01 * s01;+ }++ uint8_t scales[2];+ char2 packed[2];++#pragma unroll+ for (int b = 0; b < 2; ++b) {+ const int64_t i0 = warp_start_offset + b * vals_per_scale + lane_id_32;+ const float xi = (i0 < ne00) ? x[base_pos + i0] : 0.0f;++ float amax = fabsf(xi);+#pragma unroll+ for (int mask = 16; mask > 0; mask >>= 1) {+ amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, WARP_SIZE));+ }++ const uint8_t e = compute_e8m0_scale(amax);+ scales[b] = e;+ const float inv_s = (amax == 0.0f) ? 0.0f : __frcp_rn(ggml_cuda_e8m0_to_fp32(e));++#if CUDART_VERSION >= 12080+ const float scaled_val = xi * inv_s;++ const float val0 = __shfl_sync(0xFFFFFFFF, scaled_val, base, WARP_SIZE);+ const float val1 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 16, WARP_SIZE);+ const float val2 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 1, WARP_SIZE);+ const float val3 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 17, WARP_SIZE);++ __nv_fp4x4_e2m1 fp4_packed(make_float4(val0, val1, val2, val3));+ packed[b] = *(char2 *) &fp4_packed;+#else+ // Fallback: manual FP4 conversion using LUT+ const uint8_t q_val = ggml_cuda_float_to_fp4_e2m1(xi, inv_s);++ const uint8_t q_lo_0 = __shfl_sync(0xFFFFFFFF, q_val, base, WARP_SIZE);+ const uint8_t q_lo_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 1, WARP_SIZE);+ const uint8_t q_hi_0 = __shfl_sync(0xFFFFFFFF, q_val, base + 16, WARP_SIZE);+ const uint8_t q_hi_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 17, WARP_SIZE);++ char2 q;+ q.x = (q_hi_0 << 4) | q_lo_0;+ q.y = (q_hi_1 << 4) | q_lo_1;+ packed[b] = q;+#endif // CUDART_VERSION >= 12080+ }++ block_fp4_mmq * y = (block_fp4_mmq *) vy;+ if constexpr (scatter) {+#pragma unroll+ for (int slot = 0; slot < n_expert_used; ++slot) {+ const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];+ block_fp4_mmq * yb = y + (k_block * ne1 + i);+ char2 * yqs2 = (char2 *) yb->qs;+ if (lane_in_group == 0) {+ yqs2[quad_idx_in_block * 16 + 0 * 8 + group_id] = packed[0];+ yqs2[quad_idx_in_block * 16 + 1 * 8 + group_id] = packed[1];+ }+ if (lane_id_32 == 0) {+ yb->d4[quad_idx_in_block] = (scales[1] << 8) | scales[0];+ }+ }+ } else {+ const int64_t ib0 = blockIdx.z * ((int64_t) ne1 * (ne0 / block_fp4_mmq_size));+ block_fp4_mmq * yb = y + (ib0 + k_block * ne1 + blockIdx.x);+ char2 * yqs2 = (char2 *) yb->qs;+ if (lane_in_group == 0) {+ yqs2[quad_idx_in_block * 16 + 0 * 8 + group_id] = packed[0];+ yqs2[quad_idx_in_block * 16 + 1 * 8 + group_id] = packed[1];+ }+ if (lane_id_32 == 0) {+ yb->d4[quad_idx_in_block] = (scales[1] << 8) | scales[0];+ }+ }+ GGML_UNUSED(n_expert_used);+}++// FreeToken: 4 consecutive activations as float, from fp32, bf16 or fp16+static __device__ __forceinline__ float4 load_float4(const float * x) {+ return *(const float4 *) x;+}++static __device__ __forceinline__ float4 load_float4(const nv_bfloat16 * x) {+ const uint2 v = *(const uint2 *) x;+ const float2 lo = __bfloat1622float2(*(const nv_bfloat162 *) &v.x);+ const float2 hi = __bfloat1622float2(*(const nv_bfloat162 *) &v.y);+ return make_float4(lo.x, lo.y, hi.x, hi.y);+}++static __device__ __forceinline__ float4 load_float4(const half * x) {+ const uint2 v = *(const uint2 *) x;+ const float2 lo = __half22float2(*(const half2 *) &v.x);+ const float2 hi = __half22float2(*(const half2 *) &v.y);+ return make_float4(lo.x, lo.y, hi.x, hi.y);+}++// scatter: grid over tokens, quantize once, write to all the token's compact rows+template <mmq_q8_1_ds_layout ds_layout, bool scatter, typename src_t>+static __global__ void quantize_mmq_q8_1(+ const src_t * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) {++ constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32;+ constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32;++ const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;++ if (i0 >= ne0) {+ return;+ }++ const int64_t i00 = i0;+ ggml_cuda_pdl_sync();++ int64_t base_idx;+ if constexpr (scatter) {+ base_idx = (int64_t) blockIdx.x * s02; // one physical row per token+ } else {+ const int64_t i2 = blockIdx.z % ne2;+ const int64_t i3 = blockIdx.z / ne2;+ const int64_t i01 = ids ? ids[blockIdx.x] : blockIdx.x;+ base_idx = i3*s03 + i2*s02 + i01*s01;+ }++ block_q8_1_mmq * y = (block_q8_1_mmq *) vy;++ const int64_t k_block = i0 / QK8_1_MMQ; // column block in the channel+ const int64_t iqs = i0 % QK8_1_MMQ; // quant index in block++ // Load 4 floats per thread and calculate max. abs. value between them:+ const float4 xi = i0 < ne00 ? load_float4(x + base_idx + i00) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);+ float amax = fabsf(xi.x);+ amax = fmaxf(amax, fabsf(xi.y));+ amax = fmaxf(amax, fabsf(xi.z));+ amax = fmaxf(amax, fabsf(xi.w));++ // Exchange max. abs. value between vals_per_scale/4 threads.+#pragma unroll+ for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) {+ amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));+ }++ float sum;+ if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {+ sum = xi.x + xi.y + xi.z + xi.w;++ // Calculate sums across vals_per_sum/4 threads.+#pragma unroll+ for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) {+ sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE);+ }+ }++ const float d_inv = 127.0f / amax;+ char4 q;+ q.x = roundf(xi.x*d_inv);+ q.y = roundf(xi.y*d_inv);+ q.z = roundf(xi.z*d_inv);+ q.w = roundf(xi.w*d_inv);+ const float d = 1.0f / d_inv;++ // write the block once (normal) or to each of the token's compact rows (scatter)+ const int nwrite = scatter ? n_expert_used : 1;+#pragma unroll+ for (int slot = 0; slot < nwrite; ++slot) {+ int64_t ib;+ if constexpr (scatter) {+ const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot];+ ib = k_block*ne1 + i;+ } else {+ const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel+ ib = ib0 + k_block*ne1 + blockIdx.x;+ }++ // Write back 4 int8 values as a single 32 bit value for better memory bandwidth:+ char4 * yqs4 = (char4 *) y[ib].qs;+ yqs4[iqs/4] = q;++ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6) {+ if (iqs % 16 == 0 && iqs < 96) {+ y[ib].d2s6[2 + iqs/16] = sum;+ if (iqs % 64 == 0) {+ y[ib].d2s6[iqs/64] = d;+ }+ }+ } else if (iqs % 32 == 0) {+ if (ds_layout == MMQ_Q8_1_DS_LAYOUT_DS4) {+ y[ib].ds4[iqs/32] = make_half2(d, sum);+ } else {+ y[ib].d4[iqs/32] = d;+ }+ }+ }+ GGML_UNUSED(n_expert_used);+}++template <typename src_t>+void quantize_row_q8_1_cuda(+ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {+ GGML_ASSERT(!ids);+ GGML_ASSERT(ne0 % QK8_1 == 0);++ const uint3 ne2_fastdiv = init_fastdiv_values(ne2);++ const int64_t block_num_x = (ne0 + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;+ const dim3 num_blocks(block_num_x, ne1, ne2*ne3);+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);+ const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream);+ ggml_cuda_kernel_launch(quantize_q8_1<src_t>, launch_params, x, vy, ne00, s01, s02, s03, ne0, ne1, ne2_fastdiv);+ GGML_UNUSED(type_src0);+}++template <typename src_t>+void quantize_mmq_q8_1_cuda(+ const src_t * x, const int32_t * ids, void * vy, const ggml_type type_src0,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {+ GGML_ASSERT(ne00 % 4 == 0);+ GGML_ASSERT(ne0 % QK8_1_MMQ == 0);++ // ne1 tends to assume the highest values, therefore use it as the "x" dimension of the CUDA grid:+ const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);+ const dim3 num_blocks(ne1, block_num_y, ne2*ne3);+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);+ switch (mmq_get_q8_1_ds_layout(type_src0)) {+ case MMQ_Q8_1_DS_LAYOUT_D4:+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, false, src_t>+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ break;+ case MMQ_Q8_1_DS_LAYOUT_DS4:+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, false, src_t>+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ break;+ case MMQ_Q8_1_DS_LAYOUT_D2S6:+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, false, src_t>+ <<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ break;+ default:+ GGML_ABORT("fatal error");+ }+}++// scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)+template <typename src_t>+void quantize_scatter_mmq_q8_1_cuda(+ const src_t * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,+ const int64_t ne00, const int64_t stride_token, const int64_t ne0,+ const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) {+ GGML_ASSERT(ne00 % 4 == 0);+ GGML_ASSERT(ne0 % QK8_1_MMQ == 0);++ const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);+ const dim3 num_blocks(n_tokens, block_num_y, 1);+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);+ switch (mmq_get_q8_1_ds_layout(type_src0)) {+ case MMQ_Q8_1_DS_LAYOUT_D4:+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D4, true, src_t><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);+ break;+ case MMQ_Q8_1_DS_LAYOUT_DS4:+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4, true, src_t><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);+ break;+ case MMQ_Q8_1_DS_LAYOUT_D2S6:+ quantize_mmq_q8_1<MMQ_Q8_1_DS_LAYOUT_D2S6, true, src_t><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);+ break;+ default:+ GGML_ABORT("fatal error");+ }+}++// scatter=true reuses the quant kernels: grid over tokens, ids = inverse map (token slot -> compact row)+void quantize_scatter_mmq_fp4_cuda(+ const float * x, const int32_t * ids_src1_inv, void * vy, float * scale, const ggml_type type_src0, const bool use_aligned_float8,+ const int64_t ne00, const int64_t stride_token, const int64_t ne0,+ const int64_t n_tokens, const int64_t nrows_dst, const int n_expert_used, cudaStream_t stream) {+ GGML_ASSERT(ne0 > 0);+ if (type_src0 == GGML_TYPE_NVFP4) {+ GGML_ASSERT(scale);+ GGML_ASSERT(ne00 % QK_NVFP4 == 0);+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);+ const dim3 num_blocks(n_tokens, 1, 1);+ if (use_aligned_float8) {+ quantize_mmq_nvfp4<true, true><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, scale, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/nrows_dst, /*ne2=*/1, n_expert_used);+ } else {+ quantize_mmq_nvfp4<true, false><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, scale, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/nrows_dst, /*ne2=*/1, n_expert_used);+ }+ } else {+ GGML_ASSERT(type_src0 == GGML_TYPE_MXFP4);+ constexpr int nwarps = 8;+ constexpr int vals_per_block = nwarps * 2 * QK_MXFP4;+ const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block;+ const dim3 block_size(WARP_SIZE, nwarps, 1);+ const dim3 num_blocks(n_tokens, block_num_y, 1);+ quantize_mmq_mxfp4<true><<<num_blocks, block_size, 0, stream>>>(+ x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used);+ }+}++void quantize_mmq_fp4_cuda(+ const float * x, const int32_t * ids, void * vy, float * scale, const ggml_type type_src0, const bool use_aligned_float8,+ const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03,+ const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) {+ GGML_ASSERT(type_src0 == GGML_TYPE_MXFP4 || type_src0 == GGML_TYPE_NVFP4);+ GGML_ASSERT(ne0 > 0);++ if (type_src0 == GGML_TYPE_NVFP4) {+ GGML_ASSERT(scale);+ GGML_ASSERT(ne00 % QK_NVFP4 == 0);+ const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);+ const dim3 num_blocks(ne1, ne2 * ne3, 1);+ if (use_aligned_float8) {+ quantize_mmq_nvfp4<false, true><<<num_blocks, block_size, 0, stream>>>(+ x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ } else {+ quantize_mmq_nvfp4<false, false><<<num_blocks, block_size, 0, stream>>>(+ x, ids, vy, scale, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ }+ } else {+ GGML_ASSERT(ne0 % (2 * QK_MXFP4) == 0);++ constexpr int nwarps = 8;+ constexpr int vals_per_warp = 2 * QK_MXFP4;+ constexpr int vals_per_block = nwarps * vals_per_warp;++ const int64_t block_num_y = (ne0 + vals_per_block - 1) / vals_per_block;+ const dim3 num_blocks(ne1, block_num_y, ne2 * ne3);+ const dim3 block_size(WARP_SIZE, nwarps, 1);++ quantize_mmq_mxfp4<false><<<num_blocks, block_size, 0, stream>>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0);+ }+}++#define FREETOKEN_QUANTIZE_INSTANCES(src_t) \+ template void quantize_row_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \+ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \+ template void quantize_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, int64_t, \+ int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, cudaStream_t); \+ template void quantize_scatter_mmq_q8_1_cuda<src_t>(const src_t *, const int32_t *, void *, ggml_type, int64_t, \+ int64_t, int64_t, int64_t, int64_t, int, cudaStream_t);++FREETOKEN_QUANTIZE_INSTANCES(float)+FREETOKEN_QUANTIZE_INSTANCES(nv_bfloat16)+FREETOKEN_QUANTIZE_INSTANCES(half)diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cuhnew file mode 100644index 00000000..99283ea6--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/quantize.cuh@@ -0,0 +1,74 @@+#pragma once++#include "common.cuh"+#include "mmq.cuh"++#include <cstdint>++#define CUDA_QUANTIZE_BLOCK_SIZE 256+#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128++static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access.");+static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access.");++typedef void (*quantize_cuda_t)(+ const float * x, const int32_t * ids, void * vy,+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);++// FreeToken: src_t is float, nv_bfloat16 or half+template <typename src_t>+void quantize_row_q8_1_cuda(+ const src_t * x, const int32_t * ids, void * vy,+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);++template <typename src_t>+void quantize_mmq_q8_1_cuda(+ const src_t * x, const int32_t * ids, void * vy,+ ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,+ int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);++void quantize_mmq_fp4_cuda(const float * x,+ const int32_t * ids,+ void * vy,+ float * scale,+ ggml_type type_src0,+ bool use_aligned_float8,+ int64_t ne00,+ int64_t s01,+ int64_t s02,+ int64_t s03,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ int64_t ne3,+ cudaStream_t stream);++// quantize each token once and scatter the block to its compact rows (via the inverse map)+void quantize_scatter_mmq_fp4_cuda(const float * x,+ const int32_t * ids_src1_inv,+ void * vy,+ float * scale,+ ggml_type type_src0,+ bool use_aligned_float8,+ int64_t ne00,+ int64_t stride_token,+ int64_t ne0,+ int64_t n_tokens,+ int64_t nrows_dst,+ int n_expert_used,+ cudaStream_t stream);++template <typename src_t>+void quantize_scatter_mmq_q8_1_cuda(const src_t * x,+ const int32_t * ids_src1_inv,+ void * vy,+ ggml_type type_src0,+ int64_t ne00,+ int64_t stride_token,+ int64_t ne0,+ int64_t n_tokens,+ int64_t nrows_dst,+ int n_expert_used,+ cudaStream_t stream);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_s.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_s.cunew file mode 100644index 00000000..84ec8502--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq1_s.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ1_S);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_s.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_s.cunew file mode 100644index 00000000..583c4e5a--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_s.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ2_S);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xs.cunew file mode 100644index 00000000..edaf1560--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cunew file mode 100644index 00000000..233d9342--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_s.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_s.cunew file mode 100644index 00000000..6092dc71--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_s.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ3_S);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cunew file mode 100644index 00000000..1d5bd201--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_nl.cunew file mode 100644index 00000000..eb02fab0--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_xs.cunew file mode 100644index 00000000..1eb3b743--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-mxfp4.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-mxfp4.cunew file mode 100644index 00000000..0c1c7f29--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-mxfp4.cu@@ -0,0 +1,7 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_MXFP4);++DECL_MMQ_CASE_W4A4(GGML_TYPE_MXFP4);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-nvfp4.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-nvfp4.cunew file mode 100644index 00000000..bed8c8e2--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-nvfp4.cu@@ -0,0 +1,7 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_NVFP4);++DECL_MMQ_CASE_W4A4(GGML_TYPE_NVFP4);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q1_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q1_0.cunew file mode 100644index 00000000..f0686b0d--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q1_0.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q1_0);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_0.cunew file mode 100644index 00000000..750180e3--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_0.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q2_0);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_k.cunew file mode 100644index 00000000..6415369d--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q2_k.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q2_K);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q3_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q3_k.cunew file mode 100644index 00000000..ffb6213a--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q3_k.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q3_K);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_0.cunew file mode 100644index 00000000..0c0b0c8a--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_0.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q4_0);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_1.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_1.cunew file mode 100644index 00000000..ee67f694--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_1.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q4_1);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_k.cunew file mode 100644index 00000000..9eeb3cd7--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q4_k.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q4_K);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_0.cunew file mode 100644index 00000000..cc57fb97--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_0.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q5_0);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_1.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_1.cunew file mode 100644index 00000000..721ac790--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_1.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q5_1);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_k.cunew file mode 100644index 00000000..a2e90ffd--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q5_k.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q5_K);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q6_k.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q6_k.cunew file mode 100644index 00000000..470938fe--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q6_k.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q6_K);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q8_0.cu b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q8_0.cunew file mode 100644index 00000000..974477bb--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/template-instances/mmq-instance-q8_0.cu@@ -0,0 +1,5 @@+// This file has been autogenerated by generate_cu_files.py, do not edit manually.++#include "../mmq.cuh"++DECL_MMQ_CASE(GGML_TYPE_Q8_0);diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/unary.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/unary.cuhnew file mode 100644index 00000000..04f3af64--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/unary.cuh@@ -0,0 +1,123 @@+#pragma once+#include "common.cuh"++#define CUDA_NEG_BLOCK_SIZE 256+#define CUDA_STEP_BLOCK_SIZE 256+#define CUDA_GELU_BLOCK_SIZE 256+#define CUDA_SILU_BLOCK_SIZE 256+#define CUDA_SILU_BACK_BLOCK_SIZE 256+#define CUDA_TANH_BLOCK_SIZE 256+#define CUDA_RELU_BLOCK_SIZE 256+#define CUDA_SIGMOID_BLOCK_SIZE 256+#define CUDA_HARDSIGMOID_BLOCK_SIZE 256+#define CUDA_EXP_BLOCK_SIZE 256+#define CUDA_HARDSWISH_BLOCK_SIZE 256+#define CUDA_SQR_BLOCK_SIZE 256+#define CUDA_SQRT_BLOCK_SIZE 256+#define CUDA_SIN_BLOCK_SIZE 256+#define CUDA_COS_BLOCK_SIZE 256+#define CUDA_GLU_BLOCK_SIZE 256+#define CUDA_XIELU_BLOCK_SIZE 256++void ggml_cuda_op_abs(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_sgn(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_neg(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_step(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_gelu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_silu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_silu_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_gelu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_gelu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_tanh(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_sigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_hardsigmoid(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_exp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_hardswish(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_leaky_relu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_sqrt(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_sin(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_cos(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_log(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_expm1(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_softplus(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_elu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_floor(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_ceil(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_round(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_trunc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_reglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_geglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_swiglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_swiglu_oai(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_swiglu_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_geglu_erf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_geglu_quick(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst);++void ggml_cuda_op_unary_mul(ggml_backend_cuda_context & ctx, ggml_tensor * unary_node, ggml_tensor * mul_node);++void ggml_cuda_op_relu_sqr(ggml_backend_cuda_context & ctx, ggml_tensor * relu_node, ggml_tensor * sqr_node);++__device__ __forceinline__ float ggml_cuda_op_silu_single(float x) {+ return x / (1.0f + expf(-x));+}++__device__ __forceinline__ float ggml_cuda_op_gelu_single(float x) {+ const float GELU_COEF_A = 0.044715f;+ const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;++ return 0.5f * x * (1.0f + tanhf(SQRT_2_OVER_PI * x * (1.0f + GELU_COEF_A * x * x)));+}++__device__ __forceinline__ float ggml_cuda_op_swiglu_oai_single(float x, float g, float alpha = 1.702f, float limit = 7.0f) {+ x = fminf(x, limit);+ g = fmaxf(fminf(g, limit), -limit);++ float out_glu = x / (1.0f + expf(-x * alpha));+ out_glu = out_glu * (1.0f + g);+ return out_glu;+}++__device__ __forceinline__ float ggml_cuda_op_swiglu_clamp_single(float gate, float up, float limit) {+ gate = fminf(gate, limit);+ up = fmaxf(fminf(up, limit), -limit);++ return ggml_cuda_op_silu_single(gate) * up;+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/vecdotq.cuh b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vecdotq.cuhnew file mode 100644index 00000000..f65fe530--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vecdotq.cuh@@ -0,0 +1,1380 @@+#pragma once++#include "common.cuh"++#include <cstdint>++static __device__ __forceinline__ int get_int_b1(const void * x, const int & i32) {+ const uint8_t * x8 = (const uint8_t *) x;++ int x32 = x8[4*i32 + 0] << 0;+ x32 |= x8[4*i32 + 1] << 8;+ x32 |= x8[4*i32 + 2] << 16;+ x32 |= x8[4*i32 + 3] << 24;++ return x32;+}++static __device__ __forceinline__ int get_int_b2(const void * x, const int & i32) {+ const uint16_t * x16 = (const uint16_t *) x; // assume at least 2 byte alignment++ int x32 = x16[2*i32 + 0] << 0;+ x32 |= x16[2*i32 + 1] << 16;++ return x32;+}++static __device__ __forceinline__ int get_int_b4(const void * x, const int & i32) {+ return ((const int *) x)[i32]; // assume at least 4 byte alignment+}++// q4 contains 8 indices with 4 bit each.+// This function selects those bytes from table that are at those indices and returns them as int2.+// The first int contains the bytes with even indices in q4, the second int contains the bytes with odd indices in q4.+static __device__ __forceinline__ int2 get_int_from_table_16(const int & q4, const int8_t * table) {+#if defined(GGML_USE_HIP)+ // Load the 16-byte table into four 32-bit unsigned integers.+ const uint32_t *values = (const uint32_t *)table;++ const uint32_t q_even = q4;+ const uint32_t q_odd = (q4 >> 4);++ // Perform lookups in the lower half of the table (indices 0-7).+ uint32_t v_even_low = __builtin_amdgcn_perm(values[1], values[0], q_even & 0x07070707);+ uint32_t v_odd_low = __builtin_amdgcn_perm(values[1], values[0], q_odd & 0x07070707);++ // Perform lookups in the upper half of the table (indices 8-15).+ uint32_t v_even_high = __builtin_amdgcn_perm(values[3], values[2], q_even & 0x07070707);+ uint32_t v_odd_high = __builtin_amdgcn_perm(values[3], values[2], q_odd & 0x07070707);++ // Select between the low and high results based on the MSB of each index nibble.+ uint32_t mask_even = 0x03020100 | ((q_even & 0x08080808) >> 1);+ uint32_t res_x = __builtin_amdgcn_perm(v_even_high, v_even_low, mask_even);+ uint32_t mask_odd = 0x03020100 | ((q_odd & 0x08080808) >> 1);+ uint32_t res_y = __builtin_amdgcn_perm(v_odd_high, v_odd_low, mask_odd);++ return make_int2(res_x, res_y);+#elif !defined(GGML_USE_MUSA)+ // CUDA does not have an instruction for selecting bytes with 4 bit indices.+ // However, __byte_perm is an instruction that selects bytes with 3 bit indices that can be used instead.+ const uint32_t * table32 = (const uint32_t *) table;++ // __byte_perm selects bytes based on the lower 16 bits in its third argument.+ // Therefore, do 2 iterations over the 32 bits in q4 with 0 and 16 shift.+ // To handle the fourth bit, first call _byte_perm both for the low and the high 64 bit of table, using the low 3 bits.+ // Then, call __byte_perm again to select from the low and high bytes based on the fourth bit.+ uint32_t tmp[2];+ const uint32_t low_high_selection_indices = (0x32103210 | ((q4 & 0x88888888) >> 1));+#pragma unroll+ for (uint32_t i = 0; i < 2; ++i) {+ const uint32_t shift = 16 * i;++ const uint32_t low = __byte_perm(table32[0], table32[1], q4 >> shift);+ const uint32_t high = __byte_perm(table32[2], table32[3], q4 >> shift);+ tmp[i] = __byte_perm(low, high, low_high_selection_indices >> shift);+ }++ // tmp contains the bytes from tyble in the same order as the 4 bit indices in q4.+ // However, for the result we need ints with all even/odd 4 bit indices in q4.+ // Therefore, 2 more calls to __byte_perm to put the bytes in the correct order.+ return make_int2(__byte_perm(tmp[0], tmp[1], 0x6420), __byte_perm(tmp[0], tmp[1], 0x7531));+#else+ // Generic implementation.+ const int q0_32 = (q4 >> 0) & 0x0F0F0F0F;+ const int8_t * q0_8 = (const int8_t *) &q0_32;+ const char4 val0_8 = make_char4(+ table[q0_8[0]], table[q0_8[1]], table[q0_8[2]], table[q0_8[3]]);++ const int q1_32 = (q4 >> 4) & 0x0F0F0F0F;+ const int8_t * q1_8 = (const int8_t *) &q1_32;+ const char4 val1_8 = make_char4(+ table[q1_8[0]], table[q1_8[1]], table[q1_8[2]], table[q1_8[3]]);++ return make_int2(*((const int *) &val0_8), *((const int *) &val1_8));+#endif+}++static __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) {+ // v is a 7 bit int, with the 8th sign being encodable as popcnt+ // with xor we can "correct" the bit instead of having to mask+ const uint32_t p = __popc(v) & 1;+ const uint32_t s = v ^ p << 7;+ // broadcast over uint to allow for 0x08040201 / 0x80402010 as selectors+ return s * 0x01010101;+}++// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called+// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q++#define VDR_Q1_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism+#define VDR_Q1_0_Q8_1_MMQ 4 // Q1_0 has 128 bits (4 ints) per block++#define VDR_Q2_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism+#define VDR_Q2_0_Q8_1_MMQ 2 // Q2_0 group 64: 128 bits (4 ints) per block, 2 32-element chunks++#define VDR_Q4_0_Q8_1_MMVQ 2+#define VDR_Q4_0_Q8_1_MMQ 4++template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_impl(+ const int * v, const int * u, const float & d4, const half2 & ds8) {++ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;++ // SIMD dot product of quantized values+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);+ }++ const float2 ds8f = __half22float2(ds8);++ // second part effectively subtracts 8 from each quant value+ return d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);+}++#define VDR_Q4_1_Q8_1_MMVQ 2+#define VDR_Q4_1_Q8_1_MMQ 4++template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_impl(+ const int * v, const int * u, const half2 & dm4, const half2 & ds8) {++ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;+ const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;++ // SIMD dot product of quantized values+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi);+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi);+ }++#ifdef FAST_FP16_AVAILABLE+ const float2 tmp = __half22float2(__hmul2(dm4, ds8));+ const float d4d8 = tmp.x;+ const float m4s8 = tmp.y;+#else+ const float2 dm4f = __half22float2(dm4);+ const float2 ds8f = __half22float2(ds8);+ const float d4d8 = dm4f.x * ds8f.x;+ const float m4s8 = dm4f.y * ds8f.y;+#endif // FAST_FP16_AVAILABLE++ // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it+ return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));+}++#define VDR_Q5_0_Q8_1_MMVQ 2+#define VDR_Q5_0_Q8_1_MMQ 4++template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_impl(+ const int * vl, const int * vh, const int * u, const float & d5, const half2 & ds8) {++ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values++ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values+ }++ const float2 ds8f = __half22float2(ds8);++ // second part effectively subtracts 16 from each quant value+ return d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);+}++#define VDR_Q5_1_Q8_1_MMVQ 2+#define VDR_Q5_1_Q8_1_MMQ 4++template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impl(+ const int * vl, const int * vh, const int * u, const half2 & dm5, const half2 & ds8) {++ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits+ vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4+ vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12+ vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20+ vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28+ sumi = ggml_cuda_dp4a(vi0, u[2*i+0], sumi); // SIMD dot product of quantized values++ int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits+ vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4+ vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12+ vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20+ vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28+ sumi = ggml_cuda_dp4a(vi1, u[2*i+1], sumi); // SIMD dot product of quantized values+ }++#ifdef FAST_FP16_AVAILABLE+ const float2 tmp = __half22float2(__hmul2(dm5, ds8));+ const float d5d8 = tmp.x;+ const float m5s8 = tmp.y;+#else+ const float2 dm5f = __half22float2(dm5);+ const float2 ds8f = __half22float2(ds8);+ const float d5d8 = dm5f.x * ds8f.x;+ const float m5s8 = dm5f.y * ds8f.y;+#endif // FAST_FP16_AVAILABLE++ // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it+ return sumi*d5d8 + m5s8 / (QI5_1 / vdr);+}++#define VDR_Q8_0_Q8_1_MMVQ 2+#define VDR_Q8_0_Q8_1_MMQ 8++template <typename T, int vdr> static __device__ __forceinline__ T vec_dot_q8_0_q8_1_impl(+ const int * v, const int * u, const T & d8_0, const T & d8_1) {++ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ // SIMD dot product of quantized values+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);+ }++ return d8_0*d8_1 * ((T) sumi);+}++template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_impl(+ const int * v, const int * u, const half2 & dm8, const half2 & ds8) {++ int sumi = 0;++#pragma unroll+ for (int i = 0; i < vdr; ++i) {+ // SIMD dot product of quantized values+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);+ }++#ifdef FAST_FP16_AVAILABLE+ const float2 tmp = __half22float2(__hmul2(dm8, ds8));+ const float d8d8 = tmp.x;+ const float m8s8 = tmp.y;+#else+ const float2 dm8f = __half22float2(dm8);+ const float2 ds8f = __half22float2(ds8);+ const float d8d8 = dm8f.x * ds8f.x;+ const float m8s8 = dm8f.y * ds8f.y;+#endif // FAST_FP16_AVAILABLE++ // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it+ return sumi*d8d8 + m8s8 / (QI8_1 / vdr);+}++template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_16_q8_1_impl(+ const int * v, const int * u, const float * d8_0, const float & d8_1) {++ float sumf = 0.0f;++#pragma unroll+ for (int i0 = 0; i0 < vdr; i0 += QI8_0/2) {+ int sumi = 0;++#pragma unroll+ for (int i = i0; i < i0 + QI8_0/2; ++i) {+ // SIMD dot product of quantized values+ sumi = ggml_cuda_dp4a(v[i], u[i], sumi);+ }++ sumf += d8_0[i0/(QI8_0/2)]*sumi;+ }++ return d8_1*sumf;+}++#define VDR_MXFP4_Q8_1_MMVQ 2+#define VDR_MXFP4_Q8_1_MMQ 4++static __device__ __forceinline__ float vec_dot_mxfp4_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_mxfp4 * bq4 = (const block_mxfp4 *) vbq + kbx;++ const int * q8 = (const int *) bq8_1->qs + iqs;++ int sumi = 0;+#pragma unroll+ for (int l = 0; l < VDR_MXFP4_Q8_1_MMVQ; ++l) {+ const int aux_q4 = get_int_b1(bq4->qs, iqs + l);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_mxfp4);++ sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);+ sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);+ }++ const float d = ggml_cuda_e8m0_to_fp32(bq4->e) * 0.5f * __low2float(bq8_1->ds);+ return d * sumi;+}++#define VDR_NVFP4_Q8_1_MMVQ 4+#define VDR_NVFP4_Q8_1_MMQ 8++static __device__ __forceinline__ float vec_dot_nvfp4_q8_1(+ const void * __restrict__ vbq,+ const block_q8_1 * __restrict__ bq8_1,+ const int32_t & kbx,+ const int32_t & iqs) {++ const block_nvfp4 * bq4 = (const block_nvfp4 *) vbq + kbx;+ float sum = 0.0f;+#pragma unroll+ for (int i = 0; i < VDR_NVFP4_Q8_1_MMVQ/2; i++) {+ const int32_t iqs0 = iqs + 2*i;+ const int32_t iqs1 = iqs0 + 1;+ const int32_t is = iqs0 >> 1;+ const int2 v0 = get_int_from_table_16(get_int_b4(bq4->qs, iqs0), kvalues_mxfp4);+ const int2 v1 = get_int_from_table_16(get_int_b4(bq4->qs, iqs1), kvalues_mxfp4);+ const block_q8_1 * bq8 = bq8_1 + (is >> 1);+ const int32_t i8 = ((is & 1) << 2);++ int sumi = ggml_cuda_dp4a(v0.x, get_int_b4(bq8->qs, i8 + 0), 0);+ sumi = ggml_cuda_dp4a(v0.y, get_int_b4(bq8->qs, i8 + 2), sumi);+ sumi = ggml_cuda_dp4a(v1.x, get_int_b4(bq8->qs, i8 + 1), sumi);+ sumi = ggml_cuda_dp4a(v1.y, get_int_b4(bq8->qs, i8 + 3), sumi);++ const float d = ggml_cuda_ue4m3_to_fp32(bq4->d[is]) * __low2float(bq8->ds);+ sum += d * float(sumi);+ }++ return sum;+}+#define VDR_Q2_K_Q8_1_MMVQ 1+#define VDR_Q2_K_Q8_1_MMQ 4++// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(+ const int & v, const int * __restrict__ u, const uint8_t * __restrict__ scales,+ const half2 & dm2, const float * __restrict__ d8) {++ float sumf_d = 0.0f;+ float sumf_m = 0.0f;++#pragma unroll+ for (int i = 0; i < QR2_K; ++i) {+ const int sc = scales[2*i];++ const int vi = (v >> (2*i)) & 0x03030303;++ sumf_d += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product++ // fill int with 4x m+ int m = sc >> 4;+ m |= m << 8;+ m |= m << 16;+ sumf_m += d8[i] * ggml_cuda_dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values+ }++ const float2 dm2f = __half22float2(dm2);++ return dm2f.x*sumf_d - dm2f.y*sumf_m;+}++// contiguous v/x + u/y values+template <int ns8>+static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const half2 * dm2, const float & d8, const half2 * s8) {++ float sumf = 0.0f;+ float sumf_d8 = 0.0f;++#pragma unroll+ for (int i0 = 0; i0 < QR2_K*VDR_Q2_K_Q8_1_MMQ; i0 += QI8_1) {+ const float2 dm2f0 = __half22float2(dm2[i0/(QI8_1/2) + 0]);+ int sumi_d0 = 0;++ const float2 dm2f1 = __half22float2(dm2[i0/(QI8_1/2) + 1]);+ int sumi_d1 = 0;++#pragma unroll+ for (int i = i0; i < i0 + QI8_1/2; ++i) {+ sumi_d0 = ggml_cuda_dp4a(v[i], u[i], sumi_d0);+ }+ sumf_d8 += dm2f0.x * sumi_d0;++#pragma unroll+ for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {+ sumi_d1 = ggml_cuda_dp4a(v[i], u[i], sumi_d1);+ }+ sumf_d8 += dm2f1.x * sumi_d1;++ if (i0/QI8_1 < ns8) {+ const float2 s8f = __half22float2(s8[i0/QI8_1]);+ sumf -= dm2f0.y*s8f.x;+ sumf -= dm2f1.y*s8f.y;+ } else {+ int sumi_m0 = 0;+#pragma unroll+ for (int i = i0; i < i0 + QI8_1/2; ++i) {+ sumi_m0 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m0);+ }+ sumf_d8 -= dm2f0.y * sumi_m0;++ int sumi_m1 = 0;+#pragma unroll+ for (int i = i0 + QI8_1/2; i < i0 + QI8_1; ++i) {+ sumi_m1 = ggml_cuda_dp4a(0x01010101, u[i], sumi_m1);+ }+ sumf_d8 -= dm2f1.y * sumi_m1;+ }+ }++ return sumf + d8*sumf_d8;+}++#define VDR_Q3_K_Q8_1_MMVQ 1+#define VDR_Q3_K_Q8_1_MMQ 2++// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(+ const int & vl, const int & vh, const int * __restrict__ u, const uint8_t * __restrict__ scales,+ const int & scale_offset, const float & d3, const float * __restrict__ d8) {++ float sumf = 0.0f;++#pragma unroll+ for (int i = 0; i < QR3_K; ++i) {+ const int isc = scale_offset + 2*i;++ const int isc_low = isc % (QK_K/32);+ const int sc_shift_low = 4 * (isc / (QK_K/32));+ const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;++ const int isc_high = isc % (QK_K/64);+ const int sc_shift_high = 2 * (isc / (QK_K/64));+ const int sc_high = ((scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4;++ const int sc = (sc_low | sc_high) - 32;++ const int vil = (vl >> (2*i)) & 0x03030303;++ const int vih = ((vh >> i) << 2) & 0x04040404;++ const int vi = __vsub4(vil, vih);++ sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product+ }++ return d3 * sumf;+}++// contiguous v/x + u/y values+static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ scales,+ const float & d3, const float & d8) {++ int sumi = 0;++#pragma unroll+ for (int i0 = 0; i0 < QR3_K*VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1/2) {+ int sumi_sc = 0;++#pragma unroll+ for (int i = i0; i < i0 + QI8_1/2; ++i) {+ sumi_sc = ggml_cuda_dp4a(v[i], u[i], sumi_sc); // SIMD dot product+ }++ sumi += sumi_sc * scales[i0 / (QI8_1/2)];+ }++ return d3*d8 * sumi;+}++#define VDR_Q4_K_Q8_1_MMVQ 2+#define VDR_Q4_K_Q8_1_MMQ 8++// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const float * __restrict__ d8) {++ float sumf_d = 0.0f;+ float sumf_m = 0.0f;++#pragma unroll+ for (int i = 0; i < QR4_K; ++i) {+ const int v0i = (v[0] >> (4*i)) & 0x0F0F0F0F;+ const int v1i = (v[1] >> (4*i)) & 0x0F0F0F0F;++ const int dot1 = ggml_cuda_dp4a(v1i, u[2*i+1], ggml_cuda_dp4a(v0i, u[2*i+0], 0)); // SIMD dot product+ const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+1], ggml_cuda_dp4a(0x01010101, u[2*i+0], 0)); // sum of u++ sumf_d += d8[i] * (dot1 * sc[i]);+ sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values+ }++ const float2 dm4f = __half22float2(dm4);++ return dm4f.x*sumf_d - dm4f.y*sumf_m;+}++// contiguous v/x + u/y values+static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {++ float sumf_d = 0.0f;+ float sumf_m = 0.0f;++#pragma unroll+ for (int i = 0; i < QR4_K*VDR_Q4_K_Q8_1_MMQ/QI8_1; ++i) {+ int sumi_d = 0;++#pragma unroll+ for (int j = 0; j < QI8_1; ++j) {+ sumi_d = ggml_cuda_dp4a((v[j] >> (4*i)) & 0x0F0F0F0F, u[i*QI8_1 + j], sumi_d); // SIMD dot product+ }++ const float2 ds8f = __half22float2(ds8[i]);++ sumf_d += ds8f.x * (sc[i] * sumi_d);+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val+ }++ const float2 dm4f = __half22float2(dm4);++ return dm4f.x*sumf_d - dm4f.y*sumf_m;+}++#define VDR_Q5_K_Q8_1_MMVQ 2+#define VDR_Q5_K_Q8_1_MMQ 8++// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(+ const int * __restrict__ vl, const int * __restrict__ vh, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm5, const float * __restrict__ d8) {++ float sumf_d = 0.0f;+ float sumf_m = 0.0f;++#pragma unroll+ for (int i = 0; i < QR5_K; ++i) {+ const int vl0i = (vl[0] >> (4*i)) & 0x0F0F0F0F;+ const int vl1i = (vl[1] >> (4*i)) & 0x0F0F0F0F;++ const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;+ const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;++ const int v0i = vl0i | vh0i;+ const int v1i = vl1i | vh1i;++ const int dot1 = ggml_cuda_dp4a(v0i, u[2*i+0], ggml_cuda_dp4a(v1i, u[2*i+1], 0)); // SIMD dot product+ const int dot2 = ggml_cuda_dp4a(0x01010101, u[2*i+0], ggml_cuda_dp4a(0x01010101, u[2*i+1], 0)); // sum of u++ sumf_d += d8[i] * (dot1 * sc[i]);+ sumf_m += d8[i] * (dot2 * m[i]);++ }++ const float2 dm5f = __half22float2(dm5);++ return dm5f.x*sumf_d - dm5f.y*sumf_m;+}++// contiguous v/x + u/y values+static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const uint8_t * __restrict__ sc,+ const uint8_t * __restrict__ m, const half2 & dm4, const half2 * __restrict__ ds8) {++ float sumf_d = 0.0f;+ float sumf_m = 0.0f;++#pragma unroll+ for (int i = 0; i < QR5_K*VDR_Q5_K_Q8_1_MMQ/QI8_1; ++i) {+ int sumi_d = 0;++#pragma unroll+ for (int j = 0; j < QI8_1; ++j) {+ sumi_d = ggml_cuda_dp4a(v[i*QI8_1 + j], u[i*QI8_1 + j], sumi_d); // SIMD dot product+ }++ const float2 ds8f = __half22float2(ds8[i]);++ sumf_d += ds8f.x * (sc[i] * sumi_d);+ sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val+ }++ const float2 dm4f = __half22float2(dm4);++ return dm4f.x*sumf_d - dm4f.y*sumf_m;+}++#define VDR_Q6_K_Q8_1_MMVQ 1+#define VDR_Q6_K_Q8_1_MMQ 8++// contiguous v/x values+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(+ const int & vl, const int & vh, const int * __restrict__ u, const int8_t * __restrict__ scales,+ const float & d, const float * __restrict__ d8) {++ float sumf = 0.0f;++#pragma unroll+ for (int i = 0; i < QR6_K; ++i) {+ const int sc = scales[4*i];++ const int vil = (vl >> (4*i)) & 0x0F0F0F0F;++ const int vih = ((vh >> (4*i)) << 4) & 0x30303030;++ const int vi = __vsub4((vil | vih), 0x20202020); // vi = (vil | vih) - 32++ sumf += d8[i] * (ggml_cuda_dp4a(vi, u[i], 0) * sc); // SIMD dot product+ }++ return d*sumf;+}++// contiguous v/x + u/y values+static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(+ const int * __restrict__ v, const int * __restrict__ u, const int8_t * __restrict__ sc,+ const float & d6, const float * __restrict__ d8) {++ float sumf_d = 0.0f;++ const int sc_packed = get_int_b4(sc, 0);+ const int8_t * sc_reg = (const int8_t *) &sc_packed;++#pragma unroll+ for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {+ int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale++#pragma unroll+ for (int i = i0; i < i0 + 2; ++i) {+ sumi_d.x = ggml_cuda_dp4a(v[2*i+0], u[2*i+0], sumi_d.x); // SIMD dot product+ sumi_d.x = ggml_cuda_dp4a(v[2*i+1], u[2*i+1], sumi_d.x); // SIMD dot product++ sumi_d.y = ggml_cuda_dp4a(v[2*i+4], u[2*i+4], sumi_d.y); // SIMD dot product+ sumi_d.y = ggml_cuda_dp4a(v[2*i+5], u[2*i+5], sumi_d.y); // SIMD dot product+ }++ sumf_d += d8[i0/4] * (sc_reg[i0/2+0]*sumi_d.x + sc_reg[i0/2+1]*sumi_d.y);+ }++ return d6 * sumf_d;+}++static __device__ __forceinline__ float vec_dot_q1_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q1_0 * bq1_0 = (const block_q1_0 *) vbq + kbx;++ // Q1_0: 128 elements with ONE scale+ // Q8_1: 32 elements per block with individual scales+ // iqs selects which of the 4 chunks of 32 elements to process (0-3)++ const float d1 = bq1_0->d;+ const int16_t * qs = (const int16_t *) bq1_0->qs + iqs * 2;++ // Process only the chunk specified by iqs+ const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;++ int sumi = 0;+#pragma unroll+ for (int j = 0; j < 2; ++j) {+ const int q = qs[j];++ const int u0 = get_int_b4(bq8_1_chunk->qs, j*4+0);+ const int u1 = get_int_b4(bq8_1_chunk->qs, j*4+1);+ const int u2 = get_int_b4(bq8_1_chunk->qs, j*4+2);+ const int u3 = get_int_b4(bq8_1_chunk->qs, j*4+3);++ // unpack crumbs into nibble indices+ const int n0 = __byte_perm(0x11100100, 0x11100100, q >> 0); // [0, 1, 4, 5] [ 8, 9, 12, 13]+ const int n1 = __byte_perm(0x11100100, 0x11100100, q >> 2); // [2, 3, 6, 7] [10, 11, 14, 15]+ // unpack nibbles into byte values+ const int s0 = __byte_perm(0x01FF, 0x01FF, n0 >> 0);+ const int s1 = __byte_perm(0x01FF, 0x01FF, n1 >> 0);+ const int s2 = __byte_perm(0x01FF, 0x01FF, n0 >> 16);+ const int s3 = __byte_perm(0x01FF, 0x01FF, n1 >> 16);+ // unshuffle values+ const int v0 = __byte_perm(s0, s1, 0x5410);+ const int v1 = __byte_perm(s0, s1, 0x7632);+ const int v2 = __byte_perm(s2, s3, 0x5410);+ const int v3 = __byte_perm(s2, s3, 0x7632);++ sumi = ggml_cuda_dp4a(v0, u0, sumi);+ sumi = ggml_cuda_dp4a(v1, u1, sumi);+ sumi = ggml_cuda_dp4a(v2, u2, sumi);+ sumi = ggml_cuda_dp4a(v3, u3, sumi);+ }++ // Apply Q1_0's single scale and this chunk's Q8_1 scale+ const float d8 = __low2float(bq8_1_chunk->ds);+ return d1 * d8 * sumi;+}++static __device__ __forceinline__ float vec_dot_q2_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx;++ // Q2_0 (group 64): 64 elements with ONE scale, 2 bits per element (4 elements per byte)+ // Q8_1: 32 elements per block with individual scales+ // iqs selects which of the 2 chunks of 32 elements to process (0-1)++ const float d2 = bq2_0->d;+ const int16_t * qs = (const int16_t *) bq2_0->qs + iqs * 4;++ // Process only the chunk specified by iqs+ const block_q8_1 * bq8_1_chunk = bq8_1 + iqs;++ int sumi = 0;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int q = qs[j];+ const int u = get_int_b4(bq8_1_chunk->qs, j*2+0);+ const int v = get_int_b4(bq8_1_chunk->qs, j*2+1);++#if defined(GGML_USE_HIP)+ const uint32_t qx_indices = (q & 0x03) | ((q & 0x0C) << 6) | ((q & 0x30) << 12) | ((q & 0xC0) << 18);+ const uint32_t qy_bits = q >> 8;+ const uint32_t qy_indices = (qy_bits & 0x03) | ((qy_bits & 0x0C) << 6) | ((qy_bits & 0x30) << 12) | ((qy_bits & 0xC0) << 18);+ const int qx = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qx_indices);+ const int qy = __builtin_amdgcn_perm(0x020100FF, 0x020100FF, qy_indices);+#else+ // unpack even and odd crumbs into byte values+ const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0);+ const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2);+ // unshuffle values+ const int qx = __byte_perm(qe, qo, 0x5140);+ const int qy = __byte_perm(qe, qo, 0x7362);+#endif // defined(GGML_USE_HIP)++ sumi = ggml_cuda_dp4a(u, qx, sumi);+ sumi = ggml_cuda_dp4a(v, qy, sumi);+ }++ // Apply Q2_0's single scale and this chunk's Q8_1 scale+ const float d8 = __low2float(bq8_1_chunk->ds);+ return d2 * d8 * sumi;+}++static __device__ __forceinline__ float vec_dot_q4_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q4_0 * bq4_0 = (const block_q4_0 *) vbq + kbx;++ int v[VDR_Q4_0_Q8_1_MMVQ];+ int u[2*VDR_Q4_0_Q8_1_MMVQ];++#pragma unroll+ for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {+ v[i] = get_int_b2(bq4_0->qs, iqs + i);+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_0);+ }++ return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, bq4_0->d, bq8_1->ds);+}+++static __device__ __forceinline__ float vec_dot_q4_1_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q4_1 * bq4_1 = (const block_q4_1 *) vbq + kbx;++ int v[VDR_Q4_1_Q8_1_MMVQ];+ int u[2*VDR_Q4_1_Q8_1_MMVQ];++#pragma unroll+ for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {+ v[i] = get_int_b4(bq4_1->qs, iqs + i);+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI4_1);+ }++ return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);+}++static __device__ __forceinline__ float vec_dot_q5_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q5_0 * bq5_0 = (const block_q5_0 *) vbq + kbx;++ int vl[VDR_Q5_0_Q8_1_MMVQ];+ int vh[VDR_Q5_0_Q8_1_MMVQ];+ int u[2*VDR_Q5_0_Q8_1_MMVQ];++#pragma unroll+ for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {+ vl[i] = get_int_b2(bq5_0->qs, iqs + i);+ vh[i] = get_int_b2(bq5_0->qh, 0) >> (4 * (iqs + i));+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_0);+ }++ return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, bq5_0->d, bq8_1->ds);+}++static __device__ __forceinline__ float vec_dot_q5_1_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q5_1 * bq5_1 = (const block_q5_1 *) vbq + kbx;++ int vl[VDR_Q5_1_Q8_1_MMVQ];+ int vh[VDR_Q5_1_Q8_1_MMVQ];+ int u[2*VDR_Q5_1_Q8_1_MMVQ];++#pragma unroll+ for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {+ vl[i] = get_int_b4(bq5_1->qs, iqs + i);+ vh[i] = get_int_b4(bq5_1->qh, 0) >> (4 * (iqs + i));+ u[2*i+0] = get_int_b4(bq8_1->qs, iqs + i);+ u[2*i+1] = get_int_b4(bq8_1->qs, iqs + i + QI5_1);+ }++ return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);+}++static __device__ __forceinline__ float vec_dot_q8_0_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q8_0 * bq8_0 = (const block_q8_0 *) vbq + kbx;++ int v[VDR_Q8_0_Q8_1_MMVQ];+ int u[VDR_Q8_0_Q8_1_MMVQ];++#pragma unroll+ for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {+ v[i] = get_int_b2(bq8_0->qs, iqs + i);+ u[i] = get_int_b4(bq8_1->qs, iqs + i);+ }++ return vec_dot_q8_0_q8_1_impl<float, VDR_Q8_0_Q8_1_MMVQ>(v, u, bq8_0->d, __low2half(bq8_1->ds));+}++static __device__ __forceinline__ float vec_dot_q2_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q2_K * bq2_K = (const block_q2_K *) vbq + kbx;++ const int bq8_offset = QR2_K * (iqs / QI8_1);+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);++ const uint8_t * scales = bq2_K->scales + scale_offset;++ const int v = get_int_b4(bq2_K->qs, iqs);+ int u[QR2_K];+ float d8[QR2_K];++#pragma unroll+ for (int i = 0; i < QR2_K; ++ i) {+ u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);+ }++ return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);+}++static __device__ __forceinline__ float vec_dot_q3_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q3_K * bq3_K = (const block_q3_K *) vbq + kbx;++ const int bq8_offset = QR3_K * (iqs / (QI3_K/2));+ const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2);++ const float d = bq3_K->d;++ const int vl = get_int_b2(bq3_K->qs, iqs);++ // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted+ const int vh = ~get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset;++ int u[QR3_K];+ float d8[QR3_K];++#pragma unroll+ for (int i = 0; i < QR3_K; ++i) {+ u[i] = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + i].ds);+ }++ return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);+}++static __device__ __forceinline__ float vec_dot_q4_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q4_K * bq4_K = (const block_q4_K *) vbq + kbx;++ int v[2];+ int u[2*QR4_K];+ float d8[QR4_K];++ // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6+ const int bq8_offset = QR4_K * ((iqs/2) / (QI8_1/2));++ // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12+ // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44+ // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76+ // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108++ const int * q4 = (const int *)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));+ v[0] = q4[0];+ v[1] = q4[4];++ // branchless so nvcc can hoist this out of the ncols_dst loop+ const uint16_t * scales = (const uint16_t *)bq4_K->scales;+ const int j = bq8_offset/2;+ const int jm = j & 1;++ const uint32_t s0 = scales[jm + 0];+ const uint32_t s2 = scales[jm + 2];+ const uint32_t s4 = scales[jm + 4];++ const uint32_t hi = (uint32_t) -(int32_t) (j >= 2);++ uint16_t aux[2];+ aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi));+ aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi));+ const uint8_t * sc = (const uint8_t *)aux;+ const uint8_t * m = sc + 2;++ for (int i = 0; i < QR4_K; ++i) {+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;+ d8[i] = __low2float(bq8i->ds);++ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);+ u[2*i+0] = q8[0];+ u[2*i+1] = q8[4];+ }++ return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);+}++static __device__ __forceinline__ float vec_dot_q5_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q5_K * bq5_K = (const block_q5_K *) vbq + kbx;++ int vl[2];+ int vh[2];+ int u[2*QR5_K];+ float d8[QR5_K];++ const int bq8_offset = QR5_K * ((iqs/2) / (QI8_1/2));+ const int * ql = (const int *)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs/2)%4));+ const int * qh = (const int *)(bq5_K->qh + 4 * ((iqs/2)%4));++ vl[0] = ql[0];+ vl[1] = ql[4];++ vh[0] = qh[0] >> bq8_offset;+ vh[1] = qh[4] >> bq8_offset;++ // same as q4_K+ const uint16_t * scales = (const uint16_t *)bq5_K->scales;+ const int j = bq8_offset/2;+ const int jm = j & 1;++ const uint32_t s0 = scales[jm + 0];+ const uint32_t s2 = scales[jm + 2];+ const uint32_t s4 = scales[jm + 4];++ const uint32_t hi = (uint32_t) -(int32_t) (j >= 2);++ uint16_t aux[2];+ aux[0] = (uint16_t) (((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi));+ aux[1] = (uint16_t) (((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi));++ const uint8_t * sc = (const uint8_t *)aux;+ const uint8_t * m = sc + 2;++#pragma unroll+ for (int i = 0; i < QR5_K; ++i) {+ const block_q8_1 * bq8i = bq8_1 + bq8_offset + i;+ d8[i] = __low2float(bq8i->ds);++ const int * q8 = (const int *)bq8i->qs + ((iqs/2)%4);+ u[2*i+0] = q8[0];+ u[2*i+1] = q8[4];+ }++ return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);+}++static __device__ __forceinline__ float vec_dot_q6_K_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_q6_K * bq6_K = (const block_q6_K *) vbq + kbx;++ const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/4);+ const int scale_offset = (QI6_K/4) * (iqs / (QI6_K/2)) + (iqs % (QI6_K/2)) / (QI6_K/8);+ const int vh_shift = 2 * ((iqs % (QI6_K/2)) / (QI6_K/4));++ const int vl = get_int_b2(bq6_K->ql, iqs);+ const int vh = get_int_b2(bq6_K->qh, (QI6_K/4) * (iqs / (QI6_K/2)) + iqs % (QI6_K/4)) >> vh_shift;++ const int8_t * scales = bq6_K->scales + scale_offset;++ int u[QR6_K];+ float d8[QR6_K];++#pragma unroll+ for (int i = 0; i < QR6_K; ++i) {+ u[i] = get_int_b4(bq8_1[bq8_offset + 2*i].qs, iqs % QI8_1);+ d8[i] = __low2float(bq8_1[bq8_offset + 2*i].ds);+ }++ return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, bq6_K->d, d8);+}++#define VDR_IQ2_XXS_Q8_1_MMVQ 2+#define VDR_IQ2_XXS_Q8_1_MMQ 2++static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq + kbx;++ const int q2 = get_int_b2(bq2->qs, iqs);+ const uint8_t * aux8 = (const uint8_t *) &q2;+ const uint32_t aux32 = get_int_b2(bq2->qs, iqs + 1);++ int sumi = 0;+#pragma unroll+ for (int k0 = 0; k0 < 8; k0 += 2) {+ const uint2 grid_pos = ((const uint2*)iq2xxs_grid)[aux8[k0/2]];+ const uint32_t signs = unpack_ksigns(aux32 >> (7 * k0 / 2));++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid0 = __vsub4(grid_pos.x ^ signs0, signs0);+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, k0 + 0);+ sumi = ggml_cuda_dp4a(grid0, u0, sumi);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid1 = __vsub4(grid_pos.y ^ signs1, signs1);+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, k0 + 1);+ sumi = ggml_cuda_dp4a(grid1, u1, sumi);+ }++ const int ls = aux32 >> 27 | 1; // (scale * 2 + 1)+ sumi = sumi * ls / 8; // (sumi * scale + sumi / 2) / 4+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);+ return d * sumi;+}++#define VDR_IQ2_XS_Q8_1_MMVQ 2+#define VDR_IQ2_XS_Q8_1_MMQ 2++static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq + kbx;++ const int2 q2_packed = make_int2(get_int_b2(bq2->qs, iqs + 0), get_int_b2(bq2->qs, iqs + 1));+ const uint16_t * q2 = (const uint16_t *) &q2_packed;+ const int ls0 = bq2->scales[iqs/2] & 0x0F;+ const int ls1 = bq2->scales[iqs/2] >> 4;++ int sumi0 = 0;+ int sumi1 = 0;+#pragma unroll+ for (int l0 = 0; l0 < 8; l0 += 2) {+ const uint2 grid_pos = ((const uint2*)iq2xs_grid)[q2[l0/2] & 0x1FF];+ const uint32_t signs = unpack_ksigns(q2[l0/2] >> 9);++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);+ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);++ if (l0 < 4) {+ sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);+ sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);+ } else {+ sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);+ sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);+ }+ }+ const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;+ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);+ return d * sumi;+}++#define VDR_IQ2_S_Q8_1_MMVQ 2+#define VDR_IQ2_S_Q8_1_MMQ 2++static __device__ __forceinline__ float vec_dot_iq2_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_iq2_s * bq2 = (const block_iq2_s *) vbq + kbx;++ const int qs_packed = get_int_b2(bq2->qs, iqs/2);+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bq2->qh[iqs/2];++ const int signs_packed_32 = get_int_b2(bq2->qs, QK_K/32 + iqs/2);+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;++ const int ls0 = bq2->scales[iqs/2] & 0x0F;+ const int ls1 = bq2->scales[iqs/2] >> 4;++ int sumi0 = 0;+ int sumi1 = 0;+#pragma unroll+ for (int l0 = 0; l0 < 8; l0 += 2) {+ const int * grid_pos = (const int *)(iq2s_grid + (qs[l0/2] | ((qh << (8-l0)) & 0x300)));++ const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);+ const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);++ const int grid_l = __vsub4(grid_pos[0] ^ signs0, signs0);+ const int grid_h = __vsub4(grid_pos[1] ^ signs1, signs1);++ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);++ if (l0 < 4) {+ sumi0 = ggml_cuda_dp4a(grid_l, u0, sumi0);+ sumi0 = ggml_cuda_dp4a(grid_h, u1, sumi0);+ } else {+ sumi1 = ggml_cuda_dp4a(grid_l, u0, sumi1);+ sumi1 = ggml_cuda_dp4a(grid_h, u1, sumi1);+ }+ }+ const int sumi = (sumi0*ls0 + sumi1*ls1 + (sumi0 + sumi1)/2)/4;++ const float d = __half2float(bq2->d) * __low2float(bq8_1[iqs/2].ds);+ return d * sumi;+}++#define VDR_IQ3_XXS_Q8_1_MMVQ 2+#define VDR_IQ3_XXS_Q8_1_MMQ 2++static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_iq3_xxs * bq3 = (const block_iq3_xxs *) vbq + kbx;++ const int2 q3_packed = make_int2(get_int_b2(bq3->qs, iqs), get_int_b2(bq3->qs, iqs+1));+ const uint8_t * q3 = (const uint8_t *) &q3_packed;+ const uint32_t aux32 = get_int_b2(bq3->qs, QK_K/16 + iqs/2);++ int sumi = 0;+#pragma unroll+ for (int l0 = 0; l0 < 8; l0 += 2) {+ const int2 grid_pos = make_int2(iq3xxs_grid[q3[l0 + 0]], iq3xxs_grid[q3[l0 + 1]]);+ const uint32_t signs = unpack_ksigns(aux32 >> (7*l0/2));++ const int signs0 = __vcmpne4(signs & 0x08040201, 0);+ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);++ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);++ const int signs1 = __vcmpne4(signs & 0x80402010, 0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);++ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);++ sumi = ggml_cuda_dp4a(grid_l, u0, sumi);+ sumi = ggml_cuda_dp4a(grid_h, u1, sumi);+ }++ const int ls = aux32 >> 28;+ sumi = (ls*sumi + sumi/2)/2;+ const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);+ return d * sumi;+}++#define VDR_IQ3_S_Q8_1_MMVQ 2+#define VDR_IQ3_S_Q8_1_MMQ 2++// TODO: don't use lookup table for signs+static __device__ __forceinline__ float vec_dot_iq3_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_iq3_s * bq3 = (const block_iq3_s *) vbq + kbx;++ const int2 qs_packed = make_int2(get_int_b2(bq3->qs, iqs + 0), get_int_b2(bq3->qs, iqs + 1));+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bq3->qh[iqs/2];++ const int signs_packed_32 = get_int_b2(bq3->signs, iqs/2);+ const uint8_t * signs_packed_8 = (const uint8_t *) &signs_packed_32;++ int sumi = 0;+#pragma unroll+ for (int l0 = 0; l0 < 8; l0 += 2) {+ const int2 grid_pos = make_int2(+ iq3s_grid[qs[l0 + 0] | ((qh << (8 - l0)) & 0x100)],+ iq3s_grid[qs[l0 + 1] | ((qh << (7 - l0)) & 0x100)]);++ const int signs0 = __vcmpne4(((signs_packed_8[l0/2] & 0x03) << 7) | ((signs_packed_8[l0/2] & 0x0C) << 21), 0x00000000);+ const int signs1 = __vcmpne4(((signs_packed_8[l0/2] & 0x30) << 3) | ((signs_packed_8[l0/2] & 0xC0) << 17), 0x00000000);++ const int grid_l = __vsub4(grid_pos.x ^ signs0, signs0);+ const int grid_h = __vsub4(grid_pos.y ^ signs1, signs1);++ const int u0 = get_int_b4(bq8_1[iqs/2].qs, l0 + 0);+ const int u1 = get_int_b4(bq8_1[iqs/2].qs, l0 + 1);++ sumi = ggml_cuda_dp4a(grid_l, u0, sumi);+ sumi = ggml_cuda_dp4a(grid_h, u1, sumi);+ }++ sumi *= 1 + 2*((bq3->scales[iqs/4] >> ((iqs << 1) & 0x04)) & 0x0F);++ const float d = __half2float(bq3->d) * __low2float(bq8_1[iqs/2].ds);+ return d * sumi;+}++#define VDR_IQ1_S_Q8_1_MMVQ 1+#define VDR_IQ1_S_Q8_1_MMQ 1++static __device__ __forceinline__ float vec_dot_iq1_s_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {+ const block_iq1_s * bq1 = (const block_iq1_s *) vbq + kbx;++ const int qs_packed = get_int_b2(bq1->qs, iqs);+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ const int qh = bq1->qh[iqs];++ int sumi = 0;+#pragma unroll+ for (int l0 = 0; l0 < 8; l0 += 2) {+ const int grid = iq1s_grid_gpu[qs[l0/2] | (((qh >> 3*(l0/2)) & 0x07) << 8)];++ const int grid0 = (grid >> 0) & 0x0F0F0F0F;+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;++ const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);+ const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);++ sumi = ggml_cuda_dp4a(grid0, u0, sumi);+ sumi = ggml_cuda_dp4a(grid1, u1, sumi);+ }++ const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);+ const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f*IQ1S_DELTA/0x8000);+ const float2 ds = __half22float2(bq8_1[iqs].ds);+ return d1q * (ds.x*sumi + ds.y*delta);+}++#define VDR_IQ1_M_Q8_1_MMVQ 1+#define VDR_IQ1_M_Q8_1_MMQ 1++static __device__ __forceinline__ float vec_dot_iq1_m_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_iq1_m * bq1 = (const block_iq1_m *) vbq + kbx;++ const int qs_packed = get_int_b4(bq1->qs, iqs);+ const uint8_t * qs = (const uint8_t *) &qs_packed;++ int sumi[2] = {0};+ float sumf[2] = {0.0f};+#pragma unroll+ for (int l0 = 0; l0 < 8; l0 += 2) {+ const int qhl = bq1->qh[2*iqs + l0/4] >> (4 * ((l0/2) % 2));++ const int grid = iq1s_grid_gpu[qs[l0/2] | ((qhl & 0x07) << 8)];++ const int grid0 = (grid >> 0) & 0x0F0F0F0F;+ const int grid1 = (grid >> 4) & 0x0F0F0F0F;++ const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);+ const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);++ sumi[l0/4] = ggml_cuda_dp4a(grid0, u0, sumi[l0/4]);+ sumi[l0/4] = ggml_cuda_dp4a(grid1, u1, sumi[l0/4]);++ const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f*IQ1M_DELTA/0x08);+ int sumy = 0;+ sumy = ggml_cuda_dp4a(u0, 0x01010101, sumy);+ sumy = ggml_cuda_dp4a(u1, 0x01010101, sumy);+ sumf[l0/4] += delta*sumy;+ }++ const uint16_t * sc = (const uint16_t *) bq1->scales;++ iq1m_scale_t scale;+ scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);+ const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);++ const int tmp = sc[iqs/2] >> (6*(iqs%2));+ const int sc0 = 2*((tmp >> 0) & 0x07) + 1;+ const int sc1 = 2*((tmp >> 3) & 0x07) + 1;+ return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);+}++#define VDR_IQ4_NL_Q8_1_MMVQ 2+#define VDR_IQ4_NL_Q8_1_MMQ 4++static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_iq4_nl * bq4 = (const block_iq4_nl *) vbq + kbx;++ const int * q8 = (const int *) bq8_1->qs + iqs;++ int sumi = 0;+#pragma unroll+ for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {+ const int aux_q4 = get_int_b2(bq4->qs, iqs + l);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);++ sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi);+ sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi);+ }++ const float d = __half2float(bq4->d) * __low2float(bq8_1->ds);+ return d * sumi;+}++#define VDR_IQ4_XS_Q8_1_MMVQ 4+#define VDR_IQ4_XS_Q8_1_MMQ 4++static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1(+ const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) {++ const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq + kbx;++ int sumi = 0;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ const int aux_q4 = get_int_b4(bq4->qs, iqs + j);+ const int2 v = get_int_from_table_16(aux_q4, kvalues_iq4nl);++ const int u0 = get_int_b4(bq8_1[iqs/4].qs, j + 0);+ const int u1 = get_int_b4(bq8_1[iqs/4].qs, j + 4);++ sumi = ggml_cuda_dp4a(v.x, u0, sumi);+ sumi = ggml_cuda_dp4a(v.y, u1, sumi);+ }++ const int ls = ((bq4->scales_l[iqs/8] >> (iqs & 0x04)) & 0x0F) | (((bq4->scales_h >> (iqs/2)) & 0x03) << 4);+ sumi *= ls - 32;++ const float d = __half2float(bq4->d) * __low2float(bq8_1[iqs/4].ds);+ return d * sumi;+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/cuda.h b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/cuda.hnew file mode 100644index 00000000..323c9801--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/cuda.h@@ -0,0 +1,28 @@+#pragma once++#include <cuda_runtime.h>+#include <cuda.h>+#include <cublas_v2.h>+#include <cuda_bf16.h>+#include <cuda_fp16.h>++#ifdef GGML_USE_NCCL+#include <nccl.h>+#endif // GGML_USE_NCCL++#if CUDART_VERSION >= 11080+#include <cuda_fp8.h>+#define FP8_AVAILABLE+#endif // CUDART_VERSION >= 11080++#if CUDART_VERSION >= 12080+#include <cuda_fp4.h>+#endif // CUDART_VERSION >= 12080++#if CUDART_VERSION < 11020+#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED+#define CUBLAS_TF32_TENSOR_OP_MATH CUBLAS_TENSOR_OP_MATH+#define CUBLAS_COMPUTE_16F CUDA_R_16F+#define CUBLAS_COMPUTE_32F CUDA_R_32F+#define cublasComputeType_t cudaDataType_t+#endif // CUDART_VERSION < 11020diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/hip.h b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/hip.hnew file mode 100644index 00000000..da7e3da0--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/hip.h@@ -0,0 +1,313 @@+#pragma once++#define HIP_DISABLE_WARP_SYNC_BUILTINS 1+#include <hip/hip_runtime.h>+#include <hipblas/hipblas.h>+#include <hip/hip_fp16.h>+#include <hip/hip_bf16.h>++#ifdef GGML_USE_NCCL+#include <rccl/rccl.h>+#endif // GGML_USE_NCCL+++#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT+#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT+#define CUBLAS_OP_N HIPBLAS_OP_N+#define CUBLAS_OP_T HIPBLAS_OP_T+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS+#define CUBLAS_TF32_TENSOR_OP_MATH 0+#define CUDA_R_16F HIPBLAS_R_16F+#define CUDA_R_16BF HIPBLAS_R_16B+#define CUDA_R_32F HIPBLAS_R_32F+#define CUBLAS_SIDE_RIGHT HIPBLAS_SIDE_RIGHT+#define CUBLAS_FILL_MODE_UPPER HIPBLAS_FILL_MODE_UPPER+#define CUBLAS_DIAG_NON_UNIT HIPBLAS_DIAG_NON_UNIT+#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED hipDeviceAttributeVirtualMemoryManagementSupported+#define CU_MEM_ALLOC_GRANULARITY_RECOMMENDED hipMemAllocationGranularityRecommended+#define CU_MEM_ALLOCATION_TYPE_PINNED hipMemAllocationTypePinned+#define CU_MEM_LOCATION_TYPE_DEVICE hipMemLocationTypeDevice+#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE hipMemAccessFlagsProtReadWrite+#define CU_CHECK(fn) {hipError_t err = fn; if(err != hipSuccess) { GGML_ABORT("HipVMM Failure: %s\n", hipGetErrorString(err)); }}+#define __shfl_sync(mask, var, laneMask, width) __shfl(var, laneMask, width)+#define __shfl_up_sync(mask, var, laneMask, width) __shfl_up(var, laneMask, width)+#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)+#define __all_sync(mask, var) __all(var)+#define __any_sync(mask, var) __any(var)+#define cublasStrsmBatched hipblasStrsmBatched+#define cublasCreate hipblasCreate+#define cublasDestroy hipblasDestroy+#define cublasGemmEx hipblasGemmEx+#define cublasGemmBatchedEx hipblasGemmBatchedEx+#define cublasGemmStridedBatchedEx hipblasGemmStridedBatchedEx+#define cublasHandle_t hipblasHandle_t+#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS+#define cublasSetStream hipblasSetStream+#define cublasSgemm hipblasSgemm+#define cublasSgemmBatched hipblasSgemmBatched+#define cublasSgemmStridedBatched hipblasSgemmStridedBatched+#define cublasStatus_t hipblasStatus_t+#define cublasOperation_t hipblasOperation_t+#define cudaDevAttrCooperativeLaunch hipDeviceAttributeCooperativeLaunch+#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer+#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess+#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess+#define cudaDeviceGetAttribute hipDeviceGetAttribute+#define cudaDeviceGetPCIBusId hipDeviceGetPCIBusId+#define cudaDeviceProp hipDeviceProp_t+#define cudaDeviceSynchronize hipDeviceSynchronize+#define cudaError_t hipError_t+#define cudaErrorMemoryAllocation hipErrorOutOfMemory+#define cudaErrorPeerAccessAlreadyEnabled hipErrorPeerAccessAlreadyEnabled+#define cudaErrorPeerAccessNotEnabled hipErrorPeerAccessNotEnabled+#define cudaEventCreateWithFlags hipEventCreateWithFlags+#define cudaEventDisableTiming hipEventDisableTiming+#define cudaEventRecord hipEventRecord+#define cudaEventSynchronize hipEventSynchronize+#define cudaEvent_t hipEvent_t+#define cudaEventDestroy hipEventDestroy+#define cudaFree hipFree+#define cudaFreeHost hipHostFree+#define cudaGetDevice hipGetDevice+#define cudaGetDeviceCount hipGetDeviceCount+#define cudaGetDeviceProperties hipGetDeviceProperties+#define cudaGetErrorString hipGetErrorString+#define cudaGetLastError hipGetLastError+#define cudaHostAlloc hipHostMalloc+#define cudaHostAllocPortable hipHostMallocPortable+#define cudaHostAllocMapped hipHostMallocMapped+#define cudaHostGetDevicePointer hipHostGetDevicePointer+#define cudaHostRegister hipHostRegister+#define cudaHostRegisterPortable hipHostRegisterPortable+#define cudaHostRegisterReadOnly hipHostRegisterReadOnly+#define cudaHostUnregister hipHostUnregister+#define cudaLaunchCooperativeKernel hipLaunchCooperativeKernel+#define cudaLaunchHostFunc hipLaunchHostFunc+#define cudaMalloc hipMalloc+#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size, hipHostMallocDefault)+#define cudaMallocManaged hipMallocManaged+#define cudaMemAdvise hipMemAdvise+#define cudaMemcpy hipMemcpy+#define cudaMemcpyAsync hipMemcpyAsync+#define cudaMemcpyPeerAsync hipMemcpyPeerAsync+#define cudaMemcpy2DAsync hipMemcpy2DAsync+#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice+#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost+#define cudaMemcpyHostToDevice hipMemcpyHostToDevice+#define cudaMemcpyKind hipMemcpyKind+#define cudaMemset hipMemset+#define cudaMemsetAsync hipMemsetAsync+#define cudaMemGetInfo hipMemGetInfo+#define cudaOccupancyMaxPotentialBlockSize hipOccupancyMaxPotentialBlockSize+#define cudaSetDevice hipSetDevice+#define cuDeviceGet hipDeviceGet+#define CUdevice hipDevice_t+#define CUdeviceptr hipDeviceptr_t+#define cuMemUnmap hipMemUnmap+#define CUmemAccessDesc hipMemAccessDesc+#define cuMemAddressFree hipMemAddressFree+#define cuMemRelease hipMemRelease+#define CUmemGenericAllocationHandle hipMemGenericAllocationHandle_t+#define cuMemCreate hipMemCreate+#define cuMemAddressReserve hipMemAddressReserve+#define cuMemMap hipMemMap+#define cuMemSetAccess hipMemSetAccess+#define cuMemGetAllocationGranularity hipMemGetAllocationGranularity+#define CUmemAllocationProp hipMemAllocationProp+#define cuDeviceGetAttribute hipDeviceGetAttribute+#define cudaStreamCreateWithFlags hipStreamCreateWithFlags+#define cudaStreamDestroy hipStreamDestroy+#define cudaStreamFireAndForget hipStreamFireAndForget+#define cudaStreamNonBlocking hipStreamNonBlocking+#define cudaStreamPerThread hipStreamPerThread+#define cudaStreamSynchronize hipStreamSynchronize+#define cudaStreamWaitEvent hipStreamWaitEvent+#define cudaGraphExec_t hipGraphExec_t+#define cudaGraphNode_t hipGraphNode_t+#define cudaKernelNodeParams hipKernelNodeParams+#define cudaKernelNodeParams hipKernelNodeParams+#define cudaGraphExecDestroy hipGraphExecDestroy+#define cudaGraphLaunch hipGraphLaunch+#define cudaErrorGraphExecUpdateFailure hipErrorGraphExecUpdateFailure+#define cudaGraphExecUpdateResult hipGraphExecUpdateResult+#define cudaGraphNodeType hipGraphNodeType+#define cudaGraphNodeTypeKernel hipGraphNodeTypeKernel+#define cudaGraphInstantiate hipGraphInstantiate+#define cudaStreamEndCapture hipStreamEndCapture+#define cudaGraphDestroy hipGraphDestroy+#define cudaGraphKernelNodeSetParams hipGraphKernelNodeSetParams+#define cudaErrorInvalidDeviceFunction hipErrorInvalidDeviceFunction+#define cudaGraphKernelNodeGetParams hipGraphKernelNodeGetParams+#define cudaGraphNodeGetType hipGraphNodeGetType+#define cudaGraphGetNodes hipGraphGetNodes+#define cudaGraphExecUpdate hipGraphExecUpdate+#define cudaStreamCaptureModeRelaxed hipStreamCaptureModeRelaxed+#define cudaStreamBeginCapture hipStreamBeginCapture+#define cudaGraph_t hipGraph_t+#define cudaStream_t hipStream_t+#define cudaSuccess hipSuccess+#define cudaOccupancyMaxActiveBlocksPerMultiprocessor hipOccupancyMaxActiveBlocksPerMultiprocessor+#define cudaFuncSetAttribute hipFuncSetAttribute+#define cudaFuncAttributeMaxDynamicSharedMemorySize hipFuncAttributeMaxDynamicSharedMemorySize+#define __trap() do { abort(); __builtin_unreachable(); } while(0)+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS+#define CUBLAS_STATUS_NOT_INITIALIZED HIPBLAS_STATUS_NOT_INITIALIZED+#define CUBLAS_STATUS_ALLOC_FAILED HIPBLAS_STATUS_ALLOC_FAILED+#define CUBLAS_STATUS_INVALID_VALUE HIPBLAS_STATUS_INVALID_VALUE+#define CUBLAS_STATUS_ARCH_MISMATCH HIPBLAS_STATUS_ARCH_MISMATCH+#define CUBLAS_STATUS_MAPPING_ERROR HIPBLAS_STATUS_MAPPING_ERROR+#define CUBLAS_STATUS_EXECUTION_FAILED HIPBLAS_STATUS_EXECUTION_FAILED+#define CUBLAS_STATUS_INTERNAL_ERROR HIPBLAS_STATUS_INTERNAL_ERROR+#define CUBLAS_STATUS_NOT_SUPPORTED HIPBLAS_STATUS_NOT_SUPPORTED++#if HIP_VERSION >= 60500000+#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F+#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F+#define cublasComputeType_t hipblasComputeType_t+#define cudaDataType_t hipDataType+#else+#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F+#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_R_32F+#define cublasComputeType_t hipblasDatatype_t+#define cudaDataType_t hipblasDatatype_t+#endif // HIP_VERSION >= 6050000++#if !defined(__HIP_PLATFORM_AMD__)+#error "The HIP backend supports only AMD targets"+#endif // !defined(__HIP_PLATFORM_AMD__)++#define __CUDA_ARCH__ 1300++#if defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__)+#define GCN5+#endif // defined(__gfx900__) || defined(__gfx906__) || defined(__gfx909__) || defined(__gfx90c__)++#if defined(__gfx803__)+#define GCN4+#endif // defined(__gfx803__)++#if defined(GCN5) || defined(GCN4)+#define GCN+#endif // defined(GCN5) || defined(GCN4)++#if defined(__gfx950__)+#define CDNA4+#endif // defined(__gfx950__)++#if defined(__gfx942__)+#define CDNA3+#endif // defined(__gfx942__)++#if defined(__gfx90a__)+#define CDNA2+#endif // defined(__gfx90a__)++#if defined(__gfx908__)+#define CDNA1+#endif // defined(__gfx908__)++#if defined(CDNA4) || defined(CDNA3) || defined(CDNA2) || defined(CDNA1)+#define CDNA // For the entire family+#endif // defined(CDNA4) || defined(CDNA3) || defined(CDNA2) || defined(CDNA1)++#if defined(__GFX12__)+#define RDNA4+#endif // defined(__GFX12__)++#if defined(__GFX11__)+#define RDNA3+#endif // defined(__GFX11__)++#if defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)+#define RDNA3_5+#endif // defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)++#if defined(RDNA3) && !defined(RDNA3_5)+#define RDNA3_0+#endif // defined(RDNA3) && !defined(RDNA3_5)++#if defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \+ defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__) || defined(__gfx1037__)+#define RDNA2+#endif++#if defined(__gfx1010__) || defined(__gfx1012__)+#define RDNA1+#endif // defined(__gfx1010__) || defined(__gfx1012__)++#if defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)+#define RDNA // For the entire family+#endif // defined(RDNA4) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)++#ifndef __has_builtin+ #define __has_builtin(x) 0+#endif++typedef __hip_bfloat16 nv_bfloat16;+typedef __hip_bfloat162 nv_bfloat162;++#if HIP_VERSION >= 60300000+#include <hip/hip_fp8.h>+typedef __hip_fp8_e4m3 __nv_fp8_e4m3;+#define FP8_AVAILABLE+#endif // HIP_VERSION >= 60300000++typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));+typedef uint8_t uint8x4_t __attribute__((ext_vector_type(4)));+static __device__ __forceinline__ int __vsubss4(const int a, const int b) {+ const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);+ const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);+#if __has_builtin(__builtin_elementwise_sub_sat)+ const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);+ return reinterpret_cast<const int &>(c);+#else+ int8x4_t c;+ int16_t tmp;+#pragma unroll+ for (int i = 0; i < 4; i++) {+ tmp = va[i] - vb[i];+ if(tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();+ if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();+ c[i] = tmp;+ }+ return reinterpret_cast<int &>(c);+#endif // __has_builtin(__builtin_elementwise_sub_sat)+}++static __device__ __forceinline__ int __vsub4(const int a, const int b) {+ // do some small modifications to a and b to make the subtraction not underflow+ const unsigned int a_large = a | 0x80808080;+ const unsigned int b_small = b & 0x7f7f7f7f;+ const unsigned int result_low_7bits = a_large - b_small;++ // if two ops share the same high bit, we should flip the high bit of the result+ const unsigned int should_flip_high_1bit = (a ^ ~b) & 0x80808080;++ return result_low_7bits ^ should_flip_high_1bit;+}++static __device__ __forceinline__ unsigned int __vcmpeq4(unsigned int a, unsigned int b) {+ const uint8x4_t& va = reinterpret_cast<const uint8x4_t&>(a);+ const uint8x4_t& vb = reinterpret_cast<const uint8x4_t&>(b);+ unsigned int c;+ uint8x4_t& vc = reinterpret_cast<uint8x4_t&>(c);+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ vc[i] = va[i] == vb[i] ? 0xff : 0x00;+ }+ return c;+}++static __device__ __forceinline__ unsigned int __vcmpne4(unsigned int a, unsigned int b) {+ const unsigned int x = a ^ b;++ // any non-equal bit in a byte will set the high bit of that byte here+ // the addition will not overflow in the byte as op1 and op2 are both less than 0x80+ const unsigned int ne_low_7bits = ((x & 0x7f7f7f7f) + 0x7f7f7f7f) & 0x80808080;+ const unsigned int ne_high_1bit = x & 0x80808080;+ const unsigned int ne_any_bit = ne_low_7bits | ne_high_1bit;++ return (ne_any_bit >> 7) * 0xff;+}diff --git a/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/musa.h b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/musa.hnew file mode 100644index 00000000..22126dec--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/ggml-cuda/vendors/musa.h@@ -0,0 +1,161 @@+#pragma once++#include <musa_runtime.h>+#include <musa.h>+#include <mublas.h>+#include <musa_bf16.h>+#include <musa_fp16.h>++#ifdef __MUSA_ARCH__+#define __CUDA_ARCH__ 1300 // GGML_CUDA_CC_RUBIN+#endif // __MUSA_ARCH__++#define CUBLAS_COMPUTE_16F CUDA_R_16F+#define CUBLAS_COMPUTE_32F CUDA_R_32F+#define CUBLAS_COMPUTE_32F_FAST_16F MUBLAS_COMPUTE_32F_FAST_16F+#define CUBLAS_GEMM_DEFAULT MUBLAS_GEMM_DEFAULT+#define CUBLAS_GEMM_DEFAULT_TENSOR_OP MUBLAS_GEMM_DEFAULT+#define CUBLAS_OP_N MUBLAS_OP_N+#define CUBLAS_OP_T MUBLAS_OP_T+#define CUBLAS_DEFAULT_MATH MUBLAS_DEFAULT_MATH+#define CUBLAS_SIDE_RIGHT MUBLAS_SIDE_RIGHT+#define CUBLAS_FILL_MODE_UPPER MUBLAS_FILL_MODE_UPPER+#define CUBLAS_DIAG_NON_UNIT MUBLAS_DIAG_NON_UNIT+#define CUBLAS_STATUS_SUCCESS MUBLAS_STATUS_SUCCESS+#define CUBLAS_TF32_TENSOR_OP_MATH MUBLAS_TENSOR_OP_MATH+#define CUDA_R_16F MUSA_R_16F+#define CUDA_R_16BF MUSA_R_16BF+#define CUDA_R_32F MUSA_R_32F+#define cublasStrsmBatched mublasStrsmBatched+#define cublasComputeType_t cudaDataType_t+#define cublasCreate mublasCreate+#define cublasDestroy mublasDestroy+#define cublasGemmEx mublasGemmEx+#define cublasGemmBatchedEx mublasGemmBatchedEx+#define cublasGemmStridedBatchedEx mublasGemmStridedBatchedEx+#define cublasHandle_t mublasHandle_t+#define cublasSetMathMode mublasSetMathMode+#define cublasSetStream mublasSetStream+#define cublasSgemm mublasSgemm+#define cublasSgemmBatched mublasSgemmBatched+#define cublasSgemmStridedBatched mublasSgemmStridedBatched+#define cublasStatus_t mublasStatus_t+#define cublasOperation_t mublasOperation_t+#define cublasGetStatusString mublasGetStatusString+#define cudaDataType_t musaDataType_t+#define cudaDeviceCanAccessPeer musaDeviceCanAccessPeer+#define cudaDeviceDisablePeerAccess musaDeviceDisablePeerAccess+#define cudaDeviceEnablePeerAccess musaDeviceEnablePeerAccess+#define cudaDeviceGetPCIBusId musaDeviceGetPCIBusId+#define cudaDeviceProp musaDeviceProp+#define cudaDeviceSynchronize musaDeviceSynchronize+#define cudaDeviceGetAttribute musaDeviceGetAttribute+#define cudaError_t musaError_t+#define cudaErrorMemoryAllocation musaErrorMemoryAllocation+#define cudaErrorPeerAccessAlreadyEnabled musaErrorPeerAccessAlreadyEnabled+#define cudaErrorPeerAccessNotEnabled musaErrorPeerAccessNotEnabled+#define cudaEventCreateWithFlags musaEventCreateWithFlags+#define cudaEventDisableTiming musaEventDisableTiming+#define cudaEventRecord musaEventRecord+#define cudaEventSynchronize musaEventSynchronize+#define cudaEvent_t musaEvent_t+#define cudaEventDestroy musaEventDestroy+#define cudaFree musaFree+#define cudaFreeHost musaFreeHost+#define cudaGetDevice musaGetDevice+#define cudaGetDeviceCount musaGetDeviceCount+#define cudaGetDeviceProperties musaGetDeviceProperties+#define cudaGetErrorString musaGetErrorString+#define cudaGetLastError musaGetLastError+#define cudaHostRegister musaHostRegister+#define cudaHostRegisterPortable musaHostRegisterPortable+#define cudaHostRegisterReadOnly musaHostRegisterReadOnly+#define cudaHostUnregister musaHostUnregister+#define cudaLaunchCooperativeKernel musaLaunchCooperativeKernel+#define cudaLaunchHostFunc musaLaunchHostFunc+#define cudaMalloc musaMalloc+#define cudaMallocHost musaMallocHost+#define cudaMallocManaged musaMallocManaged+#define cudaMemcpy musaMemcpy+#define cudaMemcpyAsync musaMemcpyAsync+#define cudaMemcpyPeerAsync musaMemcpyPeerAsync+#define cudaMemcpy2DAsync musaMemcpy2DAsync+#define cudaMemcpyDeviceToDevice musaMemcpyDeviceToDevice+#define cudaMemcpyDeviceToHost musaMemcpyDeviceToHost+#define cudaMemcpyHostToDevice musaMemcpyHostToDevice+#define cudaMemcpyKind musaMemcpyKind+#define cudaMemset musaMemset+#define cudaMemsetAsync musaMemsetAsync+#define cudaMemGetInfo musaMemGetInfo+#define cudaOccupancyMaxPotentialBlockSize musaOccupancyMaxPotentialBlockSize+#define cudaSetDevice musaSetDevice+#define cudaStreamCreateWithFlags musaStreamCreateWithFlags+#define cudaStreamDestroy musaStreamDestroy+#define cudaStreamFireAndForget musaStreamFireAndForget+#define cudaStreamNonBlocking musaStreamNonBlocking+#define cudaStreamPerThread musaStreamPerThread+#define cudaStreamSynchronize musaStreamSynchronize+#define cudaStreamWaitEvent musaStreamWaitEvent+#define cudaStream_t musaStream_t+#define cudaSuccess musaSuccess++// Additional mappings for MUSA virtual memory pool+#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED MU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED+#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE MU_MEM_ACCESS_FLAGS_PROT_READWRITE+#define CU_MEM_ALLOC_GRANULARITY_RECOMMENDED MU_MEM_ALLOC_GRANULARITY_RECOMMENDED+#define CU_MEM_ALLOCATION_TYPE_PINNED MU_MEM_ALLOCATION_TYPE_PINNED+#define CU_MEM_LOCATION_TYPE_DEVICE MU_MEM_LOCATION_TYPE_DEVICE+#define CUdevice MUdevice+#define CUdeviceptr MUdeviceptr+#define CUmemAccessDesc MUmemAccessDesc+#define CUmemAllocationProp MUmemAllocationProp+#define CUmemGenericAllocationHandle MUmemGenericAllocationHandle+#define cuDeviceGet muDeviceGet+#define cuDeviceGetAttribute muDeviceGetAttribute+#define cuMemAddressFree muMemAddressFree+#define cuMemAddressReserve muMemAddressReserve+#define cuMemCreate muMemCreate+#define cuMemGetAllocationGranularity muMemGetAllocationGranularity+#define cuMemMap muMemMap+#define cuMemRelease muMemRelease+#define cuMemSetAccess muMemSetAccess+#define cuMemUnmap muMemUnmap+#define cudaDevAttrCooperativeLaunch musaDevAttrCooperativeLaunch+#define cudaFuncAttributeMaxDynamicSharedMemorySize musaFuncAttributeMaxDynamicSharedMemorySize+#define cudaFuncSetAttribute musaFuncSetAttribute+#define cudaMemcpy3DPeerParms musaMemcpy3DPeerParms+#define make_cudaExtent make_musaExtent+#define make_cudaPitchedPtr make_musaPitchedPtr++// Additional mappings for MUSA graphs+#define CUDA_SUCCESS MUSA_SUCCESS+#define CUresult MUresult+#define cuGetErrorString muGetErrorString+#define cudaErrorGraphExecUpdateFailure musaErrorGraphExecUpdateFailure+#define cudaErrorInvalidDeviceFunction musaErrorInvalidDeviceFunction+#define cudaGraphDestroy musaGraphDestroy+#define cudaGraphExecDestroy musaGraphExecDestroy+#define cudaGraphExec_t musaGraphExec_t+#define cudaGraphExecUpdate musaGraphExecUpdate+#define cudaGraphExecUpdateResult musaGraphExecUpdateResult+#define cudaGraphGetNodes musaGraphGetNodes+#define cudaGraphInstantiate musaGraphInstantiate+#define cudaGraphKernelNodeGetParams musaGraphKernelNodeGetParams+#define cudaGraphKernelNodeSetParams musaGraphKernelNodeSetParams+#define cudaGraphLaunch musaGraphLaunch+#define cudaGraphNodeGetType musaGraphNodeGetType+#define cudaGraphNode_t musaGraphNode_t+#define cudaGraphNodeType musaGraphNodeType+#define cudaGraphNodeTypeKernel musaGraphNodeTypeKernel+#define cudaGraph_t musaGraph_t+#define cudaKernelNodeParams musaKernelNodeParams+#define cudaStreamCaptureModeRelaxed musaStreamCaptureModeRelaxed+#define cudaStreamBeginCapture musaStreamBeginCapture+#define cudaStreamEndCapture musaStreamEndCapture+#define cudaStreamCaptureStatus musaStreamCaptureStatus+#define cudaStreamCaptureStatusNone musaStreamCaptureStatusNone+#define cudaStreamIsCapturing musaStreamIsCapturing+#define cudaOccupancyMaxActiveBlocksPerMultiprocessor musaOccupancyMaxActiveBlocksPerMultiprocessor++typedef __mt_bfloat16 nv_bfloat16;+typedef __mt_bfloat162 nv_bfloat162;diff --git a/python/freetoken/kernel/csrc/ggml/include/ggml-alloc.h b/python/freetoken/kernel/csrc/ggml/include/ggml-alloc.hnew file mode 100644index 00000000..81cce2e2--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/include/ggml-alloc.h@@ -0,0 +1,87 @@+#pragma once++#include "ggml.h"++#ifdef __cplusplus+extern "C" {+#endif++typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;+typedef struct ggml_backend_buffer * ggml_backend_buffer_t;+typedef struct ggml_backend * ggml_backend_t;++// Tensor allocator+struct ggml_tallocr {+ ggml_backend_buffer_t buffer;+ void * base;+ size_t alignment;+ size_t offset;+};++GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer);+GGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor);++// Graph allocator+/*+ Example usage:+ ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type());++ // optional: create a worst-case graph and reserve the buffers to avoid reallocations+ ggml_gallocr_reserve(galloc, build_graph(max_batch));++ // allocate the graph+ struct ggml_cgraph * graph = build_graph(batch);+ ggml_gallocr_alloc_graph(galloc, graph);++ printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0));++ // evaluate the graph+ ggml_backend_graph_compute(backend, graph);+*/++// special tensor flags for use with the graph allocator:+// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses+// ggml_set_output(): output tensors are never freed and never overwritten++typedef struct ggml_gallocr * ggml_gallocr_t;++GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft);+GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs);+GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);++// pre-allocate buffers from a measure graph - does not allocate or modify the graph+// call with a worst-case graph to avoid buffer reallocations+// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed+// returns false if the buffer allocation failed+// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes+GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);+GGML_API void ggml_gallocr_reserve_n_size(+ ggml_gallocr_t galloc,+ struct ggml_cgraph * graph,+ const int * node_buffer_ids,+ const int * leaf_buffer_ids,+ size_t * sizes);+GGML_API bool ggml_gallocr_reserve_n(+ ggml_gallocr_t galloc,+ struct ggml_cgraph * graph,+ const int * node_buffer_ids,+ const int * leaf_buffer_ids);++// automatic reallocation if the topology changes when using a single buffer+// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)+GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph);++GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id);++// Utils+// Create a buffer and allocate all the tensors in a ggml_context++// returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft. returns 0 on failure+GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);+// returns NULL on failure or if all tensors in ctx are already allocated or zero-sized+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);+GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/include/ggml-backend.h b/python/freetoken/kernel/csrc/ggml/include/ggml-backend.hnew file mode 100644index 00000000..7a65d812--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/include/ggml-backend.h@@ -0,0 +1,463 @@+#pragma once++#include "ggml.h"+#include "ggml-alloc.h"++#ifdef GGML_BACKEND_SHARED+# if defined(_WIN32) && !defined(__MINGW32__)+# ifdef GGML_BACKEND_BUILD+# define GGML_BACKEND_API __declspec(dllexport) extern+# else+# define GGML_BACKEND_API __declspec(dllimport) extern+# endif+# else+# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern+# endif+#else+# define GGML_BACKEND_API extern+#endif++#ifdef __cplusplus+extern "C" {+#endif++ typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;+ typedef struct ggml_backend_buffer * ggml_backend_buffer_t;+ typedef struct ggml_backend_event * ggml_backend_event_t;+ typedef struct ggml_backend * ggml_backend_t;+ typedef void * ggml_backend_graph_plan_t;+ typedef struct ggml_backend_reg * ggml_backend_reg_t;+ typedef struct ggml_backend_device * ggml_backend_dev_t;+++ //+ // Backend buffer type+ //++ GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);+ GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);+ GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);+ GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);+ GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);+ GGML_API size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);+ GGML_API size_t ggml_backend_buft_get_alloc_size_n(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);+ GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);+ GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft);++ //+ // Backend buffer+ //++ enum ggml_backend_buffer_usage {+ GGML_BACKEND_BUFFER_USAGE_ANY = 0,+ GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,+ GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,+ };++ GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);+ GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);+ GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);+ GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);+ GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);+ GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);+ GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor);+ GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);+ GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);+ GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);+ GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer);+ GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);+ GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);++ // tensor copy between different backends+ GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst);++ //+ // Backend (stream)+ //++ GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);+ GGML_API const char * ggml_backend_name(ggml_backend_t backend);+ GGML_API void ggml_backend_free(ggml_backend_t backend);++ GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);+ GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);+ GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);+ GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);++ GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+ GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);++ // "offset" refers to the offset in tensor->data for setting/getting data+ GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+ GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+ GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);++ GGML_API void ggml_backend_synchronize(ggml_backend_t backend);++ GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);+ GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);++ GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);+ GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);+ GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);++ // NOTE: will be removed, use device version instead+ GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);+ GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);+ GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);++ // asynchronous copy+ // the copy is performed after all the currently queued operations in backend_src+ // backend_dst will wait for the copy to complete before performing other operations+ // automatic fallback to sync copy if async is not supported+ GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);++ GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend);++ //+ // Events+ //++ GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device);+ GGML_API void ggml_backend_event_free(ggml_backend_event_t event);+ GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend);+ GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);+ GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event);++ //+ // Backend device+ //++ enum ggml_backend_dev_type {+ // CPU device using system memory+ GGML_BACKEND_DEVICE_TYPE_CPU,+ // GPU device using dedicated memory+ GGML_BACKEND_DEVICE_TYPE_GPU,+ // integrated GPU device using host memory+ GGML_BACKEND_DEVICE_TYPE_IGPU,+ // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)+ GGML_BACKEND_DEVICE_TYPE_ACCEL,+ // "meta" device wrapping multiple other devices for tensor parallelism+ GGML_BACKEND_DEVICE_TYPE_META,+ };++ // functionality supported by the device+ struct ggml_backend_dev_caps {+ // asynchronous operations+ bool async;+ // pinned host buffer+ bool host_buffer;+ // creating buffers from host ptr+ bool buffer_from_host_ptr;+ // event synchronization+ bool events;+ // mmap is supported for loading+ bool mmap_support;+ };++ // all the device properties+ struct ggml_backend_dev_props {+ // device name+ const char * name;+ // device description+ const char * description;+ // device free memory in bytes+ size_t memory_free;+ // device total memory in bytes+ size_t memory_total;+ // device type+ enum ggml_backend_dev_type type;+ // device id+ // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0")+ // if the id is unknown, this should be NULL+ const char * device_id;+ // device capabilities+ struct ggml_backend_dev_caps caps;+ };++ GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device);+ GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device);+ GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total);+ GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device);+ GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props);+ GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device);+ GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params);+ GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device);+ GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device);+ GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);++ GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);+ GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);+ GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);++ //+ // Backend (reg)+ //++ GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg);+ GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg);+ GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index);+ GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name);++ // Common functions that may be obtained using ggml_backend_reg_get_proc_address++ // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend)+ typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends);+ typedef void (*ggml_backend_comm_free_t)(void * comm_ctx);+ typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);++ // Split buffer type for tensor parallelism (old)+ typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);+ // Set the number of threads for the backend+ typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);+ // Get additional buffer types provided by the device (returns a NULL-terminated array)+ typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);+ // Set the abort callback for the backend+ typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);+ // Get a list of feature flags supported by the backend (returns a NULL-terminated array)+ struct ggml_backend_feature {+ const char * name;+ const char * value;+ };+ typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);++ //+ // Backend registry+ //++ GGML_API void ggml_backend_register(ggml_backend_reg_t reg);++ GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);++ // Backend (reg) enumeration+ GGML_API size_t ggml_backend_reg_count(void);+ GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index);+ GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name);++ // Device enumeration+ GGML_API size_t ggml_backend_dev_count(void);+ GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index);+ GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name);+ GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type);++ // Direct backend (stream) initialization+ // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params)+ GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params);+ // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params)+ GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params);+ // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)+ GGML_API ggml_backend_t ggml_backend_init_best(void);++ // Load a backend from a dynamic library and register it+ GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);+ // Unload a backend if loaded dynamically and unregister it+ GGML_API void ggml_backend_unload(ggml_backend_reg_t reg);+ // Load all known backends from dynamic libraries+ GGML_API void ggml_backend_load_all(void);+ GGML_API void ggml_backend_load_all_from_path(const char * dir_path);++ //+ // Backend scheduler+ //++ // The backend scheduler allows for multiple backend devices to be used together+ // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends+ // The backends are selected based on:+ // - the backend that supports the operation+ // - the location of the pre-allocated tensors (e.g. the weights)+ /*+ Example usage:++ // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned+ // preferably to run on the same backend as the buffer+ ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);++ sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true);++ // initialize buffers from a max size graph (optional)+ reserve_graph = build_graph(sched, max_batch_size);++ // manually assign nodes to a backend (optional, should not be needed in most cases)+ struct ggml_tensor * node = ggml_mul_mat(ctx, ...);+ ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);++ ggml_backend_sched_reserve(sched, reserve_graph);++ // compute+ graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation+ for (int i = 0; i < 10; ++i) {+ ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically+ }++ // if there are graph inputs:+ graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called)+ ggml_backend_sched_reset(sched); // clear the allocation of the previous graph+ ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it+ ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors+ ggml_backend_sched_graph_compute(sched, graph); // execute the graph++ // as an alternative to the above it is also possible to assign the inputs to a dedicated context and+ // allocate them statically via ggml_backend_alloc_ctx_tensors+ }+ */++ typedef struct ggml_backend_sched * ggml_backend_sched_t;++ // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback)+ // when ask == true, the scheduler wants to know if the user wants to observe this node+ // this allows the scheduler to batch nodes together in order to evaluate them in a single call+ //+ // when ask == false, the scheduler is passing the node tensor to the user for observation+ // if the user returns false, the scheduler will cancel the graph compute+ //+ typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);++ // Callback while copying input weights of a split+ // if the user returns false the scheduler simply copies the entire weight+ // the callback is called only for input weights in host buffers+ // the callback is called after all non-weight inputs of the split have been copied+ // `src` is the tensor in the previous split+ // `dst` is the copy of `src` in the split+ // `graph` is the compute graph of the split+ typedef bool (*ggml_backend_sched_copy_callback)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst, struct ggml_cgraph * graph, void * user_data);++ // Callback when the buffers reserved for a graph are not sufficient and have to be reserved again+ // it happens on the graph allocation path and costs a synchronization of the backends+ // `sched` is the scheduler+ // `graph` is the compute graph being allocated+ // `backend_ids_changed` is true if the backend assignment of the graph nodes changed since the last allocation+ // `graph_size_changed` is true if the graph size changed since the last allocation+ typedef void (*ggml_backend_sched_realloc_callback)(ggml_backend_sched_t sched, struct ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data);++ // Initialize a backend scheduler, backends with low index are given priority over backends with high index+ GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);+ GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);++ // Initialize backend buffers from a measure graph+ GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes);+ GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success++ GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched);+ GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i);++ // Get the number of splits of the last graph+ GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);+ GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);++ GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend);+ GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);++ GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);+ GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);++ // Split graph without allocating it+ GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);++ // Allocate and compute graph on the backend scheduler+ GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success+ GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);+ GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);+ GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);++ // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.+ // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.+ // The correct way to use this API is to discard the deallocated tensors and create new ones.+ GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);++ // Set a callback to be called for each resulting node during graph compute+ GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);++ // Set a callback to be called when the inputs weights of a split are being copied+ GGML_API void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data);++ // Set a callback to be called when the compute buffers of a graph are being reallocated+ GGML_API void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data);++ //+ // Meta backend+ //++#define GGML_BACKEND_META_MAX_DEVICES 16++ enum ggml_backend_meta_split_axis {+ // tensor split by tensor dimensions:+ GGML_BACKEND_SPLIT_AXIS_0 = 0,+ GGML_BACKEND_SPLIT_AXIS_1 = 1,+ GGML_BACKEND_SPLIT_AXIS_2 = 2,+ GGML_BACKEND_SPLIT_AXIS_3 = 3,++ GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends+ GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum++ // for internal bookkeeping only:+ GGML_BACKEND_SPLIT_AXIS_NONE = 98,+ GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99,+ };+ GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis);++ struct ggml_backend_meta_split_state {+ enum ggml_backend_meta_split_axis axis;++ // for tensors with axis >= 0 && axis < GGML_MAX_DIMS:+ // - each device has a slice of the tensor along the split axis+ // - most tensors have n_segments == 1 and a contiguous slice of the tensor data+ // - some tensors have an inhomogenenous data layout along the split axis,+ // those tensors are divided into segments which are each individually split across devices+ // (this usually happens when multiple tensors are fused into a single one)+ // - ne has one entry per segment and device and that segment repeats nr times,+ // in total when accounting for repetitions the segments add up to ggml_tensor::ne for that axis,+ // the outer/inner loops are over segments/devices like [seg0_dev0_r0, seg0_dev1_r0, seg0_dev0_r1, seg0_dev1_r1, seg1_dev0_r0, seg1_dev1_r0],+ // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments+ // that each need to be split individually across devices so that each device gets a slice of Q, K, and V,+ // the Q matrix can be larger than the K and V matrices so this can either be expressed as 3 segments or as 2 segments+ // where the segment for K/V repeats twice+ int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES];+ uint32_t nr[16];+ uint32_t n_segments;+ };++ // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible:+ typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata);++ // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this:+ // TODO: this looks a bit strange - a backend API creates a device. I think we should try+ // express this as a backend registry functionality instead+ GGML_API ggml_backend_dev_t ggml_backend_meta_device(+ ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud);++ //+ // Utils+ //++ struct ggml_backend_graph_copy {+ ggml_backend_buffer_t buffer;+ struct ggml_context * ctx_allocated;+ struct ggml_context * ctx_unallocated;+ struct ggml_cgraph * graph;+ };++ // Copy a graph to a different backend+ GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);+ GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);++ typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);++ // Compare the output of two backends+ GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes);++ // Tensor initialization+ GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);+ GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor);++ // CPU buffer types are always available+ GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);+ GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/include/ggml-cuda.h b/python/freetoken/kernel/csrc/ggml/include/ggml-cuda.hnew file mode 100644index 00000000..1cd81eea--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/include/ggml-cuda.h@@ -0,0 +1,47 @@+#pragma once++#include "ggml.h"+#include "ggml-backend.h"++#ifdef __cplusplus+extern "C" {+#endif++#ifdef GGML_USE_HIP+#define GGML_CUDA_NAME "ROCm"+#define GGML_CUBLAS_NAME "hipBLAS"+#elif defined(GGML_USE_MUSA)+#define GGML_CUDA_NAME "MUSA"+#define GGML_CUBLAS_NAME "muBLAS"+#else+#define GGML_CUDA_NAME "CUDA"+#define GGML_CUBLAS_NAME "cuBLAS"+#endif+#define GGML_CUDA_MAX_DEVICES 16++// backend API+GGML_BACKEND_API ggml_backend_t ggml_backend_cuda_init(int device);++GGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend);++// device buffer+GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);++// conduct allreduce operation between devices+GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends);++// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU+GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);++GGML_BACKEND_API int ggml_backend_cuda_get_device_count(void);+GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);+GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);++GGML_BACKEND_API bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);+GGML_BACKEND_API void ggml_backend_cuda_unregister_host_buffer(void * buffer);++GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cuda_reg(void);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/include/ggml.h b/python/freetoken/kernel/csrc/ggml/include/ggml.hnew file mode 100644index 00000000..23ea6437--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/include/ggml.h@@ -0,0 +1,3027 @@+#pragma once++//+// GGML Tensor Library+//+// This documentation is still a work in progress.+// If you wish some specific topics to be covered, feel free to drop a comment:+//+// https://github.com/ggml-org/whisper.cpp/issues/40+//+// ## Overview+//+// This library implements:+//+// - a set of tensor operations+// - automatic differentiation+// - basic optimization algorithms+//+// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,+// but is not limited to, the following:+//+// - linear regression+// - support vector machines+// - neural networks+//+// The library allows the user to define a certain function using the available tensor operations. This function+// definition is represented internally via a computation graph. Each tensor operation in the function definition+// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the+// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized+// using one of the available optimization algorithms.+//+// For example, here we define the function: f(x) = a*x^2 + b+//+// {+// struct ggml_init_params params = {+// .mem_size = 16*1024*1024,+// .mem_buffer = NULL,+// };+//+// // memory allocation happens here+// struct ggml_context * ctx = ggml_init(params);+//+// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);+//+// ggml_set_param(ctx, x); // x is an input variable+//+// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);+// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);+// struct ggml_tensor * x2 = ggml_mul(ctx, x, x);+// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b);+//+// ...+// }+//+// Notice that the function definition above does not involve any actual computation. The computation is performed only+// when the user explicitly requests it. For example, to compute the function's value at x = 2.0:+//+// {+// ...+//+// struct ggml_cgraph * gf = ggml_new_graph(ctx);+// ggml_build_forward_expand(gf, f);+//+// // set the input variable and parameter values+// ggml_set_f32(x, 2.0f);+// ggml_set_f32(a, 3.0f);+// ggml_set_f32(b, 4.0f);+//+// ggml_graph_compute_with_ctx(ctx, &gf, n_threads);+//+// printf("f = %f\n", ggml_get_f32_1d(f, 0));+//+// ...+// }+//+// The actual computation is performed in the ggml_graph_compute() function.+//+// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the+// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know+// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory+// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was+// actually needed.+//+// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic+// differentiation and optimization algorithms.+//+// The described approach allows to define the function graph once and then compute its forward or backward graphs+// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way+// the user can avoid the memory allocation overhead at runtime.+//+// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class+// citizens, but in theory the library can be extended to support FP8 and integer data types.+//+// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary+// and binary operations. Most of the available operations fall into one of these two categories. With time, it became+// clear that the library needs to support more complex operations. The way to support these operations is not clear+// yet, but a few examples are demonstrated in the following operations:+//+// - ggml_permute()+// - ggml_conv_1d_1s()+// - ggml_conv_1d_2s()+//+// For each tensor operator, the library implements a forward and backward computation function. The forward function+// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the+// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a+// calculus class, or watch the following video:+//+// What is Automatic Differentiation?+// https://www.youtube.com/watch?v=wG_nF1awSSY+//+//+// ## Tensor data (struct ggml_tensor)+//+// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of+// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains+// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:+//+// {+// struct ggml_tensor * c = ggml_add(ctx, a, b);+//+// assert(c->src[0] == a);+// assert(c->src[1] == b);+// }+//+// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the+// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows+// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and+// permutation. All tensor operations have to take the stride into account and not assume that the tensor is+// contiguous in memory.+//+// The data of the tensor is accessed via the "data" pointer. For example:+//+// {+// const int nx = 2;+// const int ny = 3;+//+// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny);+//+// for (int y = 0; y < ny; y++) {+// for (int x = 0; x < nx; x++) {+// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;+// }+// }+//+// ...+// }+//+// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used.+//+// ## The matrix multiplication operator (ggml_mul_mat)+//+// TODO+//+//+// ## Multi-threading+//+// TODO+//+//+// ## Overview of ggml.c+//+// TODO+//+//+// ## SIMD optimizations+//+// TODO+//+//+// ## Debugging ggml+//+// TODO+//+//++#ifdef GGML_SHARED+# if defined(_WIN32) && !defined(__MINGW32__)+# ifdef GGML_BUILD+# define GGML_API __declspec(dllexport) extern+# else+# define GGML_API __declspec(dllimport) extern+# endif+# else+# define GGML_API __attribute__ ((visibility ("default"))) extern+# endif+#else+# define GGML_API extern+#endif++// TODO: support for clang+#ifdef __GNUC__+# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint)))+#elif defined(_MSC_VER)+# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func+#else+# define GGML_DEPRECATED(func, hint) func+#endif++#ifndef __GNUC__+# define GGML_ATTRIBUTE_FORMAT(...)+#elif defined(__MINGW32__) && !defined(__clang__)+# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))+#else+# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))+#endif++#if defined(_WIN32) && !defined(_WIN32_WINNT)+# define _WIN32_WINNT 0x0A00+#endif++#include <stdbool.h>+#include <stddef.h>+#include <stdint.h>+#include <stdio.h>++#define GGML_FILE_MAGIC 0x67676d6c // "ggml"+#define GGML_FILE_VERSION 2++#define GGML_QNT_VERSION 2 // bump this on quantization format changes+#define GGML_QNT_VERSION_FACTOR 1000 // do not change this++#define GGML_MAX_DIMS 4+#define GGML_MAX_PARAMS 2048+#define GGML_MAX_SRC 10+#define GGML_MAX_N_THREADS 512+#define GGML_MAX_OP_PARAMS 64++#ifndef GGML_MAX_NAME+# define GGML_MAX_NAME 64+#endif++#define GGML_DEFAULT_N_THREADS 4+#define GGML_DEFAULT_GRAPH_SIZE 2048++#if UINTPTR_MAX == 0xFFFFFFFF+ #define GGML_MEM_ALIGN 4+#elif defined(__EMSCRIPTEN__)+// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm.+// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.)+// ref: https://github.com/ggml-org/llama.cpp/pull/18628+ #define GGML_MEM_ALIGN 8+#else+ #define GGML_MEM_ALIGN 16+#endif++#define GGML_EXIT_SUCCESS 0+#define GGML_EXIT_ABORTED 1++// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726+#define GGML_ROPE_TYPE_NORMAL 0+#define GGML_ROPE_TYPE_NEOX 2+#define GGML_ROPE_TYPE_MROPE 8+#define GGML_ROPE_TYPE_VISION 24+#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000++#define GGML_MROPE_SECTIONS 4++#define GGML_UNUSED(x) (void)(x)+#ifdef __CUDACC__+template<typename... Args>+__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {}+#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__)+#else+#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0)+#endif // __CUDACC__++#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))++#ifndef NDEBUG+# define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0)+#elif defined(__GNUC__)+# define GGML_UNREACHABLE() __builtin_unreachable()+#elif defined(_MSC_VER)+# define GGML_UNREACHABLE() __assume(0)+#else+# define GGML_UNREACHABLE() ((void) 0)+#endif++#ifdef __cplusplus+# define GGML_NORETURN [[noreturn]]+#elif defined(_MSC_VER)+# define GGML_NORETURN __declspec(noreturn)+#else+# define GGML_NORETURN _Noreturn+#endif++#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__)+#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x)++// used to copy the number of elements and stride in bytes of tensors into local variables.+// main purpose is to reduce code duplication and improve readability.+//+// example:+//+// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne);+// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb);+//+#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \+ const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \+ GGML_UNUSED(prefix##0);+#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \+ GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \+ const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \+ GGML_UNUSED(prefix##1);+#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \+ GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \+ const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \+ GGML_UNUSED(prefix##2);+#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \+ GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \+ const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \+ GGML_UNUSED(prefix##3);++#define GGML_TENSOR_UNARY_OP_LOCALS \+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)++#define GGML_TENSOR_BINARY_OP_LOCALS \+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)++#define GGML_TENSOR_TERNARY_OP_LOCALS \+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \+ GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)++#define GGML_TENSOR_BINARY_OP_LOCALS01 \+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \+ GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \+ GGML_TENSOR_LOCALS(size_t, nb1, src1, nb)++#ifdef __cplusplus+extern "C" {+#endif++ // Function type used in fatal error callbacks+ typedef void (*ggml_abort_callback_t)(const char * error_message);++ // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)+ // Returns the old callback for chaining+ GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback);++ GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4)+ GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...);++ enum ggml_status {+ GGML_STATUS_ALLOC_FAILED = -2,+ GGML_STATUS_FAILED = -1,+ GGML_STATUS_SUCCESS = 0,+ GGML_STATUS_ABORTED = 1,+ };++ // get ggml_status name string+ GGML_API const char * ggml_status_to_string(enum ggml_status status);++ // ieee 754-2008 half-precision float16+ // todo: make this not an integral type+ typedef uint16_t ggml_fp16_t;+ GGML_API float ggml_fp16_to_fp32(ggml_fp16_t);+ GGML_API ggml_fp16_t ggml_fp32_to_fp16(float);+ GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t);+ GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t);++ // google brain half-precision bfloat16+ typedef struct { uint16_t bits; } ggml_bf16_t;+ GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);+ GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16+ GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);+ GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);+ GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);++ struct ggml_object;+ struct ggml_context;+ struct ggml_cgraph;++ // NOTE: always add types at the end of the enum to keep backward compatibility+ enum ggml_type {+ GGML_TYPE_F32 = 0,+ GGML_TYPE_F16 = 1,+ GGML_TYPE_Q4_0 = 2,+ GGML_TYPE_Q4_1 = 3,+ // GGML_TYPE_Q4_2 = 4, support has been removed+ // GGML_TYPE_Q4_3 = 5, support has been removed+ GGML_TYPE_Q5_0 = 6,+ GGML_TYPE_Q5_1 = 7,+ GGML_TYPE_Q8_0 = 8,+ GGML_TYPE_Q8_1 = 9,+ GGML_TYPE_Q2_K = 10,+ GGML_TYPE_Q3_K = 11,+ GGML_TYPE_Q4_K = 12,+ GGML_TYPE_Q5_K = 13,+ GGML_TYPE_Q6_K = 14,+ GGML_TYPE_Q8_K = 15,+ GGML_TYPE_IQ2_XXS = 16,+ GGML_TYPE_IQ2_XS = 17,+ GGML_TYPE_IQ3_XXS = 18,+ GGML_TYPE_IQ1_S = 19,+ GGML_TYPE_IQ4_NL = 20,+ GGML_TYPE_IQ3_S = 21,+ GGML_TYPE_IQ2_S = 22,+ GGML_TYPE_IQ4_XS = 23,+ GGML_TYPE_I8 = 24,+ GGML_TYPE_I16 = 25,+ GGML_TYPE_I32 = 26,+ GGML_TYPE_I64 = 27,+ GGML_TYPE_F64 = 28,+ GGML_TYPE_IQ1_M = 29,+ GGML_TYPE_BF16 = 30,+ // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files+ // GGML_TYPE_Q4_0_4_8 = 32,+ // GGML_TYPE_Q4_0_8_8 = 33,+ GGML_TYPE_TQ1_0 = 34,+ GGML_TYPE_TQ2_0 = 35,+ // GGML_TYPE_IQ4_NL_4_4 = 36,+ // GGML_TYPE_IQ4_NL_4_8 = 37,+ // GGML_TYPE_IQ4_NL_8_8 = 38,+ GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)+ GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)+ GGML_TYPE_Q1_0 = 41,+ GGML_TYPE_Q2_0 = 42,+ GGML_TYPE_COUNT = 43,+ };++ // [TAG_GGML_PREC]+ // this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op+ // the declared types can be:+ // - result accumulation type+ // - source tensor data representation type+ // - etc.+ // the precision parameters are stored as ggml_tensor.op_params to the respective ops+ enum ggml_prec {+ GGML_PREC_UNDEFINED = 0,+ GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED+ GGML_PREC_F32 = 10,+ GGML_PREC_BF16 = 15,+ GGML_PREC_F16 = 20,+ GGML_PREC_Q8 = 30,+ GGML_PREC_Q4 = 40,+ };++ // op hint+ enum ggml_op_hint {+ GGML_HINT_NONE = 0,+ GGML_HINT_SRC0_IS_HADAMARD = 1,+ };++ // model file types+ enum ggml_ftype {+ GGML_FTYPE_UNKNOWN = -1,+ GGML_FTYPE_ALL_F32 = 0,+ GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16+ GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors+ GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors+ GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors+ GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors+ GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors+ GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors+ };++ // available tensor operations:+ enum ggml_op {+ GGML_OP_NONE = 0,++ GGML_OP_DUP,+ GGML_OP_ADD,+ GGML_OP_ADD_ID,+ GGML_OP_ADD1,+ GGML_OP_ACC,+ GGML_OP_SUB,+ GGML_OP_MUL,+ GGML_OP_DIV,+ GGML_OP_SQR,+ GGML_OP_SQRT,+ GGML_OP_LOG,+ GGML_OP_SIN,+ GGML_OP_COS,+ GGML_OP_SUM,+ GGML_OP_SUM_ROWS,+ GGML_OP_CUMSUM,+ GGML_OP_MEAN,+ GGML_OP_ARGMAX,+ GGML_OP_COUNT_EQUAL,+ GGML_OP_REPEAT,+ GGML_OP_REPEAT_BACK,+ GGML_OP_CONCAT,+ GGML_OP_SILU_BACK,+ GGML_OP_NORM, // normalize+ GGML_OP_RMS_NORM,+ GGML_OP_RMS_NORM_BACK,+ GGML_OP_GROUP_NORM,+ GGML_OP_L2_NORM,++ GGML_OP_MUL_MAT,+ GGML_OP_MUL_MAT_ID,+ GGML_OP_OUT_PROD,++ GGML_OP_SCALE,+ GGML_OP_SET,+ GGML_OP_CPY,+ GGML_OP_CONT,+ GGML_OP_RESHAPE,+ GGML_OP_VIEW,+ GGML_OP_PERMUTE,+ GGML_OP_TRANSPOSE,+ GGML_OP_GET_ROWS,+ GGML_OP_GET_ROWS_BACK,+ GGML_OP_SET_ROWS,+ GGML_OP_DIAG,+ GGML_OP_DIAG_MASK_INF,+ GGML_OP_DIAG_MASK_ZERO,+ GGML_OP_SOFT_MAX,+ GGML_OP_SOFT_MAX_BACK,+ GGML_OP_ROPE,+ GGML_OP_ROPE_BACK,+ GGML_OP_CLAMP,+ GGML_OP_CONV_TRANSPOSE_1D,+ GGML_OP_IM2COL,+ GGML_OP_IM2COL_BACK,+ GGML_OP_IM2COL_3D,+ GGML_OP_COL2IM_1D,+ GGML_OP_CONV_2D,+ GGML_OP_CONV_3D,+ GGML_OP_CONV_2D_DW,+ GGML_OP_CONV_TRANSPOSE_2D,+ GGML_OP_POOL_1D,+ GGML_OP_POOL_2D,+ GGML_OP_POOL_2D_BACK,+ GGML_OP_UPSCALE,+ GGML_OP_PAD,+ GGML_OP_PAD_REFLECT_1D,+ GGML_OP_ROLL,+ GGML_OP_ARANGE,+ GGML_OP_TIMESTEP_EMBEDDING,+ GGML_OP_ARGSORT,+ GGML_OP_TOP_K,+ GGML_OP_LEAKY_RELU,+ GGML_OP_TRI,+ GGML_OP_FILL,++ GGML_OP_FLASH_ATTN_EXT,+ GGML_OP_FLASH_ATTN_BACK,+ GGML_OP_SSM_CONV,+ GGML_OP_SSM_SCAN,+ GGML_OP_WIN_PART,+ GGML_OP_WIN_UNPART,+ GGML_OP_GET_REL_POS,+ GGML_OP_ADD_REL_POS,+ GGML_OP_RWKV_WKV6,+ GGML_OP_GATED_LINEAR_ATTN,+ GGML_OP_RWKV_WKV7,+ GGML_OP_SOLVE_TRI,+ GGML_OP_GATED_DELTA_NET,+ GGML_OP_LIGHTNING_INDEXER,+ GGML_OP_DSV4_HC_COMB,+ GGML_OP_DSV4_HC_PRE,+ GGML_OP_DSV4_HC_POST,++ GGML_OP_UNARY,++ GGML_OP_MAP_CUSTOM1,+ GGML_OP_MAP_CUSTOM2,+ GGML_OP_MAP_CUSTOM3,++ GGML_OP_CUSTOM,++ GGML_OP_CROSS_ENTROPY_LOSS,+ GGML_OP_CROSS_ENTROPY_LOSS_BACK,+ GGML_OP_OPT_STEP_ADAMW,+ GGML_OP_OPT_STEP_SGD,++ GGML_OP_GLU,++ GGML_OP_COUNT,+ };++ enum ggml_unary_op {+ GGML_UNARY_OP_ABS,+ GGML_UNARY_OP_SGN,+ GGML_UNARY_OP_NEG,+ GGML_UNARY_OP_STEP,+ GGML_UNARY_OP_TANH,+ GGML_UNARY_OP_ELU,+ GGML_UNARY_OP_RELU,+ GGML_UNARY_OP_SIGMOID,+ GGML_UNARY_OP_GELU,+ GGML_UNARY_OP_GELU_QUICK,+ GGML_UNARY_OP_SILU,+ GGML_UNARY_OP_HARDSWISH,+ GGML_UNARY_OP_HARDSIGMOID,+ GGML_UNARY_OP_EXP,+ GGML_UNARY_OP_EXPM1,+ GGML_UNARY_OP_SOFTPLUS,+ GGML_UNARY_OP_GELU_ERF,+ GGML_UNARY_OP_XIELU,+ GGML_UNARY_OP_FLOOR,+ GGML_UNARY_OP_CEIL,+ GGML_UNARY_OP_ROUND,+ GGML_UNARY_OP_TRUNC,++ GGML_UNARY_OP_COUNT,+ };++ enum ggml_glu_op {+ GGML_GLU_OP_REGLU,+ GGML_GLU_OP_GEGLU,+ GGML_GLU_OP_SWIGLU,+ GGML_GLU_OP_SWIGLU_OAI,+ GGML_GLU_OP_GEGLU_ERF,+ GGML_GLU_OP_GEGLU_QUICK,+ GGML_GLU_OP_SWIGLU_CLAMP,++ GGML_GLU_OP_COUNT,+ };++ enum ggml_object_type {+ GGML_OBJECT_TYPE_TENSOR,+ GGML_OBJECT_TYPE_GRAPH,+ GGML_OBJECT_TYPE_WORK_BUFFER+ };++ enum ggml_log_level {+ GGML_LOG_LEVEL_NONE = 0,+ GGML_LOG_LEVEL_DEBUG = 1,+ GGML_LOG_LEVEL_INFO = 2,+ GGML_LOG_LEVEL_WARN = 3,+ GGML_LOG_LEVEL_ERROR = 4,+ GGML_LOG_LEVEL_CONT = 5, // continue previous log+ };++ // this tensor...+ enum ggml_tensor_flag {+ GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph+ GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph+ GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters+ GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up)+ GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed+ };++ enum ggml_tri_type {+ GGML_TRI_TYPE_UPPER_DIAG = 0,+ GGML_TRI_TYPE_UPPER = 1,+ GGML_TRI_TYPE_LOWER_DIAG = 2,+ GGML_TRI_TYPE_LOWER = 3+ };++ struct ggml_init_params {+ // memory pool+ size_t mem_size; // bytes+ void * mem_buffer; // if NULL, memory will be allocated internally+ bool no_alloc; // don't allocate memory for the tensor data+ };++ // n-dimensional tensor+ struct ggml_tensor {+ enum ggml_type type;++ struct ggml_backend_buffer * buffer;++ int64_t ne[GGML_MAX_DIMS]; // number of elements+ size_t nb[GGML_MAX_DIMS]; // stride in bytes:+ // nb[0] = ggml_type_size(type)+ // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding+ // nb[i] = nb[i-1] * ne[i-1]++ // compute data+ enum ggml_op op;++ // op params - allocated as int32_t for alignment+ int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)];++ int32_t flags;++ struct ggml_tensor * src[GGML_MAX_SRC];++ // source tensor and offset for views+ struct ggml_tensor * view_src;+ size_t view_offs;++ void * data;++ char name[GGML_MAX_NAME];++ void * extra; // extra things e.g. for ggml-cuda.cu++ char padding[8];+ };++ static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);++ // Abort callback+ // If not NULL, called before ggml computation+ // If it returns true, the computation is aborted+ typedef bool (*ggml_abort_callback)(void * data);+++ //+ // GUID+ //++ // GUID types+ typedef uint8_t ggml_guid[16];+ typedef ggml_guid * ggml_guid_t;++ GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b);++ // misc++ GGML_API const char * ggml_version(void);+ GGML_API const char * ggml_commit(void);++ GGML_API void ggml_time_init(void); // call this once at the beginning of the program+ GGML_API int64_t ggml_time_ms(void);+ GGML_API int64_t ggml_time_us(void);+ GGML_API int64_t ggml_cycles(void);+ GGML_API int64_t ggml_cycles_per_ms(void);++ // accepts a UTF-8 path, even on Windows+ GGML_API FILE * ggml_fopen(const char * fname, const char * mode);++ GGML_API void ggml_print_object (const struct ggml_object * obj);+ GGML_API void ggml_print_objects(const struct ggml_context * ctx);++ GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);+ GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor);+ GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor);+ GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN++ GGML_API int64_t ggml_blck_size(enum ggml_type type);+ GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block+ GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row++ GGML_DEPRECATED(+ GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float+ "use ggml_row_size() instead");++ GGML_API const char * ggml_type_name(enum ggml_type type);+ GGML_API const char * ggml_op_name (enum ggml_op op);+ GGML_API const char * ggml_op_symbol(enum ggml_op op);++ GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op);+ GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op);+ GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name++ GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);++ GGML_API bool ggml_is_quantized(enum ggml_type type);++ // TODO: temporary until model loading of ggml examples is refactored+ GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);++ GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_view (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);+ GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars++ // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation)+ GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous()+ GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1+ GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2++ GGML_API bool ggml_is_contiguous_to_1(const struct ggml_tensor * tensor); // contiguous for dims < 1+ GGML_API bool ggml_is_contiguous_to_2(const struct ggml_tensor * tensor); // contiguous for dims < 2+ GGML_API bool ggml_is_contiguous_to_3(const struct ggml_tensor * tensor); // contiguous for dims < 3++ // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok)+ GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor);++ // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN+ GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor);++ // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements+ GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor);++ GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1);+ GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1);++ GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1);++ // use this to compute the memory overhead of a tensor+ GGML_API size_t ggml_tensor_overhead(void);++ GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes);++ // main++ GGML_API struct ggml_context * ggml_init (struct ggml_init_params params);+ GGML_API void ggml_reset(struct ggml_context * ctx);+ GGML_API void ggml_free (struct ggml_context * ctx);++ GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);++ GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);+ GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);++ GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);+ GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);+ GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);++ GGML_API struct ggml_tensor * ggml_new_tensor(+ struct ggml_context * ctx,+ enum ggml_type type,+ int n_dims,+ const int64_t *ne);++ GGML_API struct ggml_tensor * ggml_new_tensor_1d(+ struct ggml_context * ctx,+ enum ggml_type type,+ int64_t ne0);++ GGML_API struct ggml_tensor * ggml_new_tensor_2d(+ struct ggml_context * ctx,+ enum ggml_type type,+ int64_t ne0,+ int64_t ne1);++ GGML_API struct ggml_tensor * ggml_new_tensor_3d(+ struct ggml_context * ctx,+ enum ggml_type type,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2);++ GGML_API struct ggml_tensor * ggml_new_tensor_4d(+ struct ggml_context * ctx,+ enum ggml_type type,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ int64_t ne3);++ GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes);++ GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);+ GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);++ // Context tensor enumeration and lookup+ GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);+ GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);+ GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);++ // Converts a flat index into coordinates+ GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);++ GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);+ GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor);++ GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);+ GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);++ GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);+ GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);+ GGML_ATTRIBUTE_FORMAT(2, 3)+ GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);++ // Tensor flags+ GGML_API void ggml_set_input(struct ggml_tensor * tensor);+ GGML_API void ggml_set_output(struct ggml_tensor * tensor);+ GGML_API void ggml_set_param(struct ggml_tensor * tensor);+ GGML_API void ggml_set_loss(struct ggml_tensor * tensor);++ //+ // operations on tensors with backpropagation+ //++ GGML_API struct ggml_tensor * ggml_dup(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_dup_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_add(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_add_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_add_cast(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ enum ggml_type type);++ // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]]+ GGML_API struct ggml_tensor * ggml_add_id(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * ids);++ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b),+ "use ggml_add instead");++ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b),+ "use ggml_add_inplace instead");++ // dst = a+ // view(dst, nb1, nb2, nb3, offset) += b+ // return dst+ GGML_API struct ggml_tensor * ggml_acc(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ size_t nb1,+ size_t nb2,+ size_t nb3,+ size_t offset);++ GGML_API struct ggml_tensor * ggml_acc_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ size_t nb1,+ size_t nb2,+ size_t nb3,+ size_t offset);++ GGML_API struct ggml_tensor * ggml_sub(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_sub_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_mul(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_mul_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_div(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_div_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_sqr(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sqr_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sqrt(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sqrt_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_log(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_log_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_expm1(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_expm1_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_softplus(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_softplus_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sin(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sin_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_cos(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_cos_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // return scalar+ GGML_API struct ggml_tensor * ggml_sum(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]+ GGML_API struct ggml_tensor * ggml_sum_rows(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_cumsum(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // mean along rows+ GGML_API struct ggml_tensor * ggml_mean(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // argmax along rows+ GGML_API struct ggml_tensor * ggml_argmax(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // count number of equal elements in a and b+ GGML_API struct ggml_tensor * ggml_count_equal(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ // if a is the same shape as b, and a is not parameter, return a+ // otherwise, return a new tensor: repeat(a) to fit in b+ GGML_API struct ggml_tensor * ggml_repeat(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ // repeat a to the specified shape+ GGML_API struct ggml_tensor * ggml_repeat_4d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ int64_t ne3);++ // sums repetitions in a into shape of b+ GGML_API struct ggml_tensor * ggml_repeat_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride++ // concat a and b along dim+ // used in stable-diffusion+ GGML_API struct ggml_tensor * ggml_concat(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int dim);++ GGML_API struct ggml_tensor * ggml_abs(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_abs_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sgn(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sgn_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_neg(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_neg_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_step(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_step_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_tanh(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_tanh_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_elu(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_elu_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_relu(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_leaky_relu(+ struct ggml_context * ctx,+ struct ggml_tensor * a, float negative_slope, bool inplace);++ GGML_API struct ggml_tensor * ggml_relu_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sigmoid(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_sigmoid_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_gelu(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_gelu_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // GELU using erf (error function) when possible+ // some backends may fallback to approximation based on Abramowitz and Stegun formula+ GGML_API struct ggml_tensor * ggml_gelu_erf(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_gelu_erf_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_gelu_quick(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_gelu_quick_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_silu(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_silu_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // a - dy+ // b - x+ GGML_API struct ggml_tensor * ggml_silu_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ // hardswish(x) = x * relu6(x + 3) / 6+ GGML_API struct ggml_tensor * ggml_hardswish(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // hardsigmoid(x) = relu6(x + 3) / 6+ GGML_API struct ggml_tensor * ggml_hardsigmoid(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_exp(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_exp_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_floor(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_floor_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_ceil(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_ceil_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_round(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_round_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ /**+ * Truncates the fractional part of each element in the tensor (towards zero).+ * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0+ * Similar to std::trunc in C/C++.+ */++ GGML_API struct ggml_tensor * ggml_trunc(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_trunc_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++++ // xIELU activation function+ // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0)+ // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions+ // that constrain the positive and negative source alpha values respectively+ GGML_API struct ggml_tensor * ggml_xielu(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float alpha_n,+ float alpha_p,+ float beta,+ float eps);++ // gated linear unit ops+ // A: n columns, r rows,+ // result is n / 2 columns, r rows,+ // expects gate in second half of row, unless swapped is true+ GGML_API struct ggml_tensor * ggml_glu(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_glu_op op,+ bool swapped);++ GGML_API struct ggml_tensor * ggml_reglu(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_reglu_swapped(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_geglu(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_geglu_swapped(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_swiglu(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_swiglu_swapped(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_geglu_erf(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_geglu_erf_swapped(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_geglu_quick(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ GGML_API struct ggml_tensor * ggml_geglu_quick_swapped(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // A: n columns, r rows,+ // B: n columns, r rows,+ GGML_API struct ggml_tensor * ggml_glu_split(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ enum ggml_glu_op op);++ GGML_API struct ggml_tensor * ggml_reglu_split(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_geglu_split(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_swiglu_split(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_geglu_erf_split(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_geglu_quick_split(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ GGML_API struct ggml_tensor * ggml_swiglu_oai(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ float alpha,+ float limit);++ GGML_API struct ggml_tensor * ggml_swiglu_clamp(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ float limit);++ // normalize along rows+ GGML_API struct ggml_tensor * ggml_norm(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float eps);++ GGML_API struct ggml_tensor * ggml_norm_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float eps);++ GGML_API struct ggml_tensor * ggml_rms_norm(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float eps);++ GGML_API struct ggml_tensor * ggml_rms_norm_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float eps);++ // group normalize along ne0*ne1*n_groups+ // used in stable-diffusion+ GGML_API struct ggml_tensor * ggml_group_norm(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int n_groups,+ float eps);++ GGML_API struct ggml_tensor * ggml_group_norm_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int n_groups,+ float eps);++ // l2 normalize along rows+ // used in rwkv v7+ GGML_API struct ggml_tensor * ggml_l2_norm(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float eps);++ GGML_API struct ggml_tensor * ggml_l2_norm_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float eps);++ // a - x+ // b - dy+ GGML_API struct ggml_tensor * ggml_rms_norm_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ float eps);++ // [TAG_GGML_PREC]+ // set the minimum required accumulator type for the implementation to use during the compute+ // for example:+ // - GGML_PREC_F32 - requires accumulation of the results in F32+ // - GGML_PREC_BF16 - can accumulate the results in BF16, F32+ // - GGML_PREC_F16 - can accumulate the results in F16, F32+ // - GGML_PREC_Q8 - not allowed+ // - GGML_PREC_Q4 - not allowed+ //+ // return false on faliure+ GGML_API bool ggml_prec_set_acc(+ struct ggml_tensor * a,+ enum ggml_prec prec);++ // [TAG_GGML_PREC]+ // set the smallest rank that the implementation can use to internally convert the src[idx] data to+ // ranks in decreasing order:+ // - GGML_PREC_F32 - GGML_TYPE_F32+ // - GGML_PREC_BF16 - GGML_TYPE_BF16+ // - GGML_PREC_F16 - GGML_TYPE_F16,+ // - GGML_PREC_Q8 - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc.+ // - GGML_PREC_Q4 - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc.+ //+ // for example:+ // - ggml_prec_set_src(a, GGML_PREC_Q8, 1):+ // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0+ // - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4+ // - ggml_prec_set_src(a, GGML_PREC_Q4, 1):+ // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc.+ //+ // return false on faliure+ GGML_API bool ggml_prec_set_src(+ struct ggml_tensor * a,+ enum ggml_prec prec,+ int idx);++ // A: k columns, n rows => [ne03, ne02, n, k]+ // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]+ // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]+ GGML_API struct ggml_tensor * ggml_mul_mat(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ // change the precision of a matrix multiplication+ // set to GGML_PREC_F32 for higher precision (useful for phi-2)+ GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec(+ struct ggml_tensor * a,+ enum ggml_prec prec),+ "use ggml_prec_set_acc() instead");++ // change the hint of a matrix multiplication+ GGML_API void ggml_mul_mat_set_hint(+ struct ggml_tensor * a,+ enum ggml_op_hint hint);++ // indirect matrix multiplication+ //+ // [TAG_MUL_MAT_ID_DUPLICATE_ID]+ // note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`+ // (i.e. duplicate experts per token are not guaranteed to be supported in all backends)+ // TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533+ GGML_API struct ggml_tensor * ggml_mul_mat_id(+ struct ggml_context * ctx,+ struct ggml_tensor * as,+ struct ggml_tensor * b,+ struct ggml_tensor * ids);++ // A: m columns, n rows,+ // B: p columns, n rows,+ // result is m columns, p rows+ GGML_API struct ggml_tensor * ggml_out_prod(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ //+ // operations on tensors without backpropagation+ //++ GGML_API struct ggml_tensor * ggml_scale(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float s);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_scale_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float s);++ // x = s * a + b+ GGML_API struct ggml_tensor * ggml_scale_bias(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float s,+ float b);++ GGML_API struct ggml_tensor * ggml_scale_bias_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float s,+ float b);++ // b -> view(a,offset,nb1,nb2,3), return modified a+ GGML_API struct ggml_tensor * ggml_set(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ size_t nb1,+ size_t nb2,+ size_t nb3,+ size_t offset); // in bytes++ // b -> view(a,offset,nb1,nb2,3), return view(a)+ GGML_API struct ggml_tensor * ggml_set_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ size_t nb1,+ size_t nb2,+ size_t nb3,+ size_t offset); // in bytes++ GGML_API struct ggml_tensor * ggml_set_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ size_t offset); // in bytes++ GGML_API struct ggml_tensor * ggml_set_1d_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ size_t offset); // in bytes++ // b -> view(a,offset,nb1,nb2,3), return modified a+ GGML_API struct ggml_tensor * ggml_set_2d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ size_t nb1,+ size_t offset); // in bytes++ // b -> view(a,offset,nb1,nb2,3), return view(a)+ GGML_API struct ggml_tensor * ggml_set_2d_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ size_t nb1,+ size_t offset); // in bytes++ // a -> b, return view(b)+ GGML_API struct ggml_tensor * ggml_cpy(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ // note: casting from f32 to i32 will discard the fractional part+ GGML_API struct ggml_tensor * ggml_cast(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_type type);++ // make contiguous+ GGML_API struct ggml_tensor * ggml_cont(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // make contiguous, with new shape+ GGML_API struct ggml_tensor * ggml_cont_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0);++ GGML_API struct ggml_tensor * ggml_cont_2d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1);++ GGML_API struct ggml_tensor * ggml_cont_3d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2);++ GGML_API struct ggml_tensor * ggml_cont_4d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ int64_t ne3);++ // return view(a), b specifies the new shape+ // TODO: when we start computing gradient, make a copy instead of view+ GGML_API struct ggml_tensor * ggml_reshape(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ // return view(a)+ // TODO: when we start computing gradient, make a copy instead of view+ GGML_API struct ggml_tensor * ggml_reshape_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0);++ GGML_API struct ggml_tensor * ggml_reshape_2d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1);++ // return view(a)+ // TODO: when we start computing gradient, make a copy instead of view+ GGML_API struct ggml_tensor * ggml_reshape_3d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2);++ GGML_API struct ggml_tensor * ggml_reshape_4d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ int64_t ne3);++ // offset in bytes+ GGML_API struct ggml_tensor * ggml_view_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ size_t offset);++ GGML_API struct ggml_tensor * ggml_view_2d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ size_t nb1, // row stride in bytes+ size_t offset);++ GGML_API struct ggml_tensor * ggml_view_3d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ size_t nb1, // row stride in bytes+ size_t nb2, // slice stride in bytes+ size_t offset);++ GGML_API struct ggml_tensor * ggml_view_4d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ int64_t ne3,+ size_t nb1, // row stride in bytes+ size_t nb2, // slice stride in bytes+ size_t nb3,+ size_t offset);++ GGML_API struct ggml_tensor * ggml_permute(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int axis0,+ int axis1,+ int axis2,+ int axis3);++ // alias for ggml_permute(ctx, a, 1, 0, 2, 3)+ GGML_API struct ggml_tensor * ggml_transpose(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // supports 4D a:+ // a [n_embd, ne1, ne2, ne3]+ // b I32 [n_rows, ne2, ne3, 1]+ //+ // return [n_embd, n_rows, ne2, ne3]+ GGML_API struct ggml_tensor * ggml_get_rows(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // data+ struct ggml_tensor * b); // row indices++ GGML_API struct ggml_tensor * ggml_get_rows_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // gradients of ggml_get_rows result+ struct ggml_tensor * b, // row indices+ struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape++ // a TD [n_embd, ne1, ne2, ne3]+ // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3+ // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1)+ //+ // undefined behavior if destination rows overlap+ //+ // broadcast:+ // ne2 % ne11 == 0+ // ne3 % ne12 == 0+ //+ // return view(a)+ GGML_API struct ggml_tensor * ggml_set_rows(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // destination+ struct ggml_tensor * b, // source+ struct ggml_tensor * c); // row indices++ GGML_API struct ggml_tensor * ggml_diag(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // set elements above the diagonal to -INF+ GGML_API struct ggml_tensor * ggml_diag_mask_inf(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int n_past);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int n_past);++ // set elements above the diagonal to 0+ GGML_API struct ggml_tensor * ggml_diag_mask_zero(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int n_past);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int n_past);++ GGML_API struct ggml_tensor * ggml_clamp(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float min,+ float max);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_clamp_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float min,+ float max);++ GGML_API struct ggml_tensor * ggml_soft_max(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_soft_max_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a);++ // a [ne0, ne01, ne02, ne03]+ // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional+ //+ // broadcast:+ // ne02 % ne12 == 0+ // ne03 % ne13 == 0+ //+ // fused soft_max(a*scale + mask*(ALiBi slope))+ // max_bias = 0.0f for no ALiBi+ GGML_API struct ggml_tensor * ggml_soft_max_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * mask,+ float scale,+ float max_bias);++ GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * mask,+ float scale,+ float max_bias);++ GGML_API void ggml_soft_max_add_sinks(+ struct ggml_tensor * a,+ struct ggml_tensor * sinks);++ GGML_API struct ggml_tensor * ggml_soft_max_ext_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ float scale,+ float max_bias);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ float scale,+ float max_bias);++ // rotary position embedding+ // if (mode & 1) - skip n_past elements (NOT SUPPORTED)+ // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style+ //+ // b is an int32 vector with size a->ne[2], it contains the positions+ GGML_API struct ggml_tensor * ggml_rope(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int n_dims,+ int mode);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_rope_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int n_dims,+ int mode);++ // RoPE operations with extended options+ // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token]+ // b is an int32 vector with size n_token+ // c is freq factors (e.g. phi3-128k), (optional)+ // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi+ //+ // pseudo-code for computing theta:+ // for i in [0, n_dims/2):+ // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims);+ // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c+ // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here+ //+ // other params are used by YaRN RoPE scaling, these default values will disable YaRN:+ // freq_scale = 1.0f+ // ext_factor = 0.0f+ // attn_factor = 1.0f+ // beta_fast = 0.0f+ // beta_slow = 0.0f+ //+ // example:+ // (marking: c = cos, s = sin, 0 = unrotated)+ // given a single head with size = 8 --> [00000000]+ // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000]+ // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs]+ // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000]+ // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss]+ GGML_API struct ggml_tensor * ggml_rope_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * c,+ int n_dims,+ int mode,+ int n_ctx_orig,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow);++ // multi-dimensional RoPE, for Qwen-VL and similar vision models+ // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX+ // sections specify how many dimensions to rotate in each section:+ // section length is equivalent to number of cos/sin pairs, NOT the number of dims+ // (i.e. sum of 4 sections are expected to be n_dims/2)+ // last sections can be 0, means ignored+ // all other options are identical to ggml_rope_ext+ //+ // important note:+ // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION+ // if you need normal ordering, there are 2 methods:+ // (1) split the tensor manually using ggml_view+ // (2) permute the weight upon conversion+ // - for VISION, n_dims must be head_size/2+ //+ // example M-RoPE:+ // given sections = [t=4, y=2, x=2, 0]+ // given a single head with size = 18 --> [000000000000000000]+ // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering)+ // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering)+ // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section+ // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section+ //+ // example vision RoPE:+ // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored)+ // given a single head with size = 8 --> [00000000]+ // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx]+ // other values of n_dims are untested and is undefined behavior+ // note: unlike MROPE, the theta for each dim is computed differently for each section+ // in other words, idx used for theta: [0123] for y section, then [0123] for x section+ GGML_API struct ggml_tensor * ggml_rope_multi(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * c,+ int n_dims,+ int sections[GGML_MROPE_SECTIONS],+ int mode,+ int n_ctx_orig,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow);++ // in-place, returns view(a)+ GGML_API struct ggml_tensor * ggml_rope_ext_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * c,+ int n_dims,+ int mode,+ int n_ctx_orig,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow);++ GGML_API struct ggml_tensor * ggml_rope_multi_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * c,+ int n_dims,+ int sections[GGML_MROPE_SECTIONS],+ int mode,+ int n_ctx_orig,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow);++ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int n_dims,+ int mode,+ int n_ctx_orig,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow),+ "use ggml_rope_ext instead");++ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int n_dims,+ int mode,+ int n_ctx_orig,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow),+ "use ggml_rope_ext_inplace instead");++ // compute correction dims for YaRN RoPE scaling+ GGML_API void ggml_rope_yarn_corr_dims(+ int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]);++ // rotary position embedding backward, i.e compute dx from dy+ // a - dy+ GGML_API struct ggml_tensor * ggml_rope_ext_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // gradients of ggml_rope result+ struct ggml_tensor * b, // positions+ struct ggml_tensor * c, // freq factors+ int n_dims,+ int mode,+ int n_ctx_orig,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow);++ GGML_API struct ggml_tensor * ggml_rope_multi_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * c,+ int n_dims,+ int sections[4],+ int mode,+ int n_ctx_orig,+ float freq_base,+ float freq_scale,+ float ext_factor,+ float attn_factor,+ float beta_fast,+ float beta_slow);++ // set the offset dims for RoPE+ // a must be GGML_OP_ROPE or GGML_OP_ROPE_BACK+ // vision RoPE is not supported+ // example: (marking: x = rotated, 0 = unrotated)+ // n_embd = 10, n_dims = 4, offset = 2 --> [00xxxx0000]+ GGML_API struct ggml_tensor * ggml_rope_set_offset(+ struct ggml_tensor * a,+ int n_offs);++ // im2col+ // converts data into a format that effectively results in a convolution when combined with matrix multiplication+ GGML_API struct ggml_tensor * ggml_im2col(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // data+ int s0, // stride dimension 0+ int s1, // stride dimension 1+ int p0, // padding dimension 0+ int p1, // padding dimension 1+ int d0, // dilation dimension 0+ int d1, // dilation dimension 1+ bool is_2D,+ enum ggml_type dst_type);++ GGML_API struct ggml_tensor * ggml_im2col_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // gradient of im2col output+ int64_t * ne, // shape of im2col input+ int s0, // stride dimension 0+ int s1, // stride dimension 1+ int p0, // padding dimension 0+ int p1, // padding dimension 1+ int d0, // dilation dimension 0+ int d1, // dilation dimension 1+ bool is_2D);++ // col2im_1d: scatter-add GEMM columns back to 1D signal+ // a: [K*OC, T_in] (columns from matmul, K = a->ne[0]/OC)+ // result: [T_out, OC] where T_out = (T_in - 1)*s0 + K - 2*p0+ GGML_API struct ggml_tensor * ggml_col2im_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // columns [K*OC, T_in]+ int s0, // stride+ int oc, // output channels+ int p0); // padding to crop from both sides++ GGML_API struct ggml_tensor * ggml_conv_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // data+ int s0, // stride+ int p0, // padding+ int d0); // dilation++ // conv_1d with padding = half+ // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d)+ GGML_API struct ggml_tensor* ggml_conv_1d_ph(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // data+ int s, // stride+ int d); // dilation++ // depthwise+ // TODO: this is very likely wrong for some cases! - needs more testing+ GGML_API struct ggml_tensor * ggml_conv_1d_dw(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // data+ int s0, // stride+ int p0, // padding+ int d0); // dilation++ GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // data+ int s0, // stride+ int d0); // dilation++ GGML_API struct ggml_tensor * ggml_conv_transpose_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // data+ int s0, // stride+ int p0, // padding+ int d0); // dilation++ GGML_API struct ggml_tensor * ggml_conv_2d(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // data+ int s0, // stride dimension 0+ int s1, // stride dimension 1+ int p0, // padding dimension 0+ int p1, // padding dimension 1+ int d0, // dilation dimension 0+ int d1); // dilation dimension 1++ GGML_API struct ggml_tensor * ggml_im2col_3d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int64_t IC,+ int s0, // stride width+ int s1, // stride height+ int s2, // stride depth+ int p0, // padding width+ int p1, // padding height+ int p2, // padding depth+ int d0, // dilation width+ int d1, // dilation height+ int d2, // dilation depth+ enum ggml_type dst_type);++ // a: [OC*IC, KD, KH, KW]+ // b: [N*IC, ID, IH, IW]+ // result: [N*OC, OD, OH, OW]+ GGML_API struct ggml_tensor * ggml_conv_3d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int64_t IC,+ int s0, // stride width+ int s1, // stride height+ int s2, // stride depth+ int p0, // padding width+ int p1, // padding height+ int p2, // padding depth+ int d0, // dilation width+ int d1, // dilation height+ int d2 // dilation depth+ );++ // kernel size is a->ne[0] x a->ne[1]+ // stride is equal to kernel size+ // padding is zero+ // example:+ // a: 16 16 3 768+ // b: 1024 1024 3 1+ // res: 64 64 768 1+ // used in sam+ GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ // kernel size is a->ne[0] x a->ne[1]+ // stride is 1+ // padding is half+ // example:+ // a: 3 3 256 256+ // b: 64 64 256 1+ // res: 64 64 256 1+ // used in sam+ GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b);++ // depthwise (via im2col and mul_mat)+ GGML_API struct ggml_tensor * ggml_conv_2d_dw(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel+ struct ggml_tensor * b, // data+ int s0, // stride dimension 0+ int s1, // stride dimension 1+ int p0, // padding dimension 0+ int p1, // padding dimension 1+ int d0, // dilation dimension 0+ int d1); // dilation dimension 1++ // Depthwise 2D convolution+ // may be faster than ggml_conv_2d_dw, but not available in all backends+ // a: KW KH 1 C convolution kernel+ // b: W H C N input data+ // res: W_out H_out C N+ GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int stride0,+ int stride1,+ int pad0,+ int pad1,+ int dilation0,+ int dilation1);++ GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ int stride);++ GGML_API struct ggml_tensor * ggml_conv_2d_direct(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC]+ struct ggml_tensor * b, // input data [W, H, C, N]+ int s0, // stride dimension 0+ int s1, // stride dimension 1+ int p0, // padding dimension 0+ int p1, // padding dimension 1+ int d0, // dilation dimension 0+ int d1); // dilation dimension 1++ GGML_API struct ggml_tensor * ggml_conv_3d_direct(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC]+ struct ggml_tensor * b, // input [W, H, D, C * N]+ int s0, // stride+ int s1,+ int s2,+ int p0, // padding+ int p1,+ int p2,+ int d0, // dilation+ int d1,+ int d2,+ int n_channels,+ int n_batch,+ int n_channels_out);++ enum ggml_op_pool {+ GGML_OP_POOL_MAX,+ GGML_OP_POOL_AVG,+ GGML_OP_POOL_COUNT,+ };++ GGML_API struct ggml_tensor * ggml_pool_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_op_pool op,+ int k0, // kernel size+ int s0, // stride+ int p0); // padding++ // the result will have 2*p0 padding for the first dimension+ // and 2*p1 padding for the second dimension+ GGML_API struct ggml_tensor * ggml_pool_2d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_op_pool op,+ int k0,+ int k1,+ int s0,+ int s1,+ float p0,+ float p1);++ GGML_API struct ggml_tensor * ggml_pool_2d_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * af, // "a"/input used in forward pass+ enum ggml_op_pool op,+ int k0,+ int k1,+ int s0,+ int s1,+ float p0,+ float p1);++ enum ggml_scale_mode {+ GGML_SCALE_MODE_NEAREST = 0,+ GGML_SCALE_MODE_BILINEAR = 1,+ GGML_SCALE_MODE_BICUBIC = 2,++ GGML_SCALE_MODE_COUNT+ };++ enum ggml_scale_flag {+ GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8),+ GGML_SCALE_FLAG_ANTIALIAS = (1 << 9),+ };++ // interpolate+ // multiplies ne0 and ne1 by scale factor+ GGML_API struct ggml_tensor * ggml_upscale(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int scale_factor,+ enum ggml_scale_mode mode);++ // interpolate+ // interpolate scale to specified dimensions+ GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int ne0,+ int ne1,+ int ne2,+ int ne3,+ enum ggml_scale_mode mode),+ "use ggml_interpolate instead");++ // Up- or downsamples the input to the specified size.+ // 2D scale modes (eg. bilinear) are applied to the first two dimensions.+ GGML_API struct ggml_tensor * ggml_interpolate(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ int64_t ne3,+ uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...]++ // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]+ GGML_API struct ggml_tensor * ggml_pad(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int p0,+ int p1,+ int p2,+ int p3);++ // pad each dimension with values on the other side of the torus (looping around)+ GGML_API struct ggml_tensor * ggml_pad_circular(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int p0,+ int p1,+ int p2,+ int p3);++ GGML_API struct ggml_tensor * ggml_pad_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int lp0,+ int rp0,+ int lp1,+ int rp1,+ int lp2,+ int rp2,+ int lp3,+ int rp3+ );++ // pad each dimension with values on the other side of the torus (looping around)+ GGML_API struct ggml_tensor * ggml_pad_ext_circular(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int lp0,+ int rp0,+ int lp1,+ int rp1,+ int lp2,+ int rp2,+ int lp3,+ int rp3);++ // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c]+ GGML_API struct ggml_tensor * ggml_pad_reflect_1d(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int p0,+ int p1);++ // Move tensor elements by an offset given for each dimension. Elements that+ // are shifted beyond the last position are wrapped around to the beginning.+ GGML_API struct ggml_tensor * ggml_roll(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int shift0,+ int shift1,+ int shift2,+ int shift3);++ // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing+ // zeroes everywhere outside the masked area+ GGML_API struct ggml_tensor * ggml_tri(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_tri_type type);++ // Fill tensor a with constant c+ GGML_API struct ggml_tensor * ggml_fill(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float c);++ GGML_API struct ggml_tensor * ggml_fill_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ float c);++ // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151+ // timesteps: [N,]+ // return: [N, dim]+ GGML_API struct ggml_tensor * ggml_timestep_embedding(+ struct ggml_context * ctx,+ struct ggml_tensor * timesteps,+ int dim,+ int max_period);++ // sort rows+ enum ggml_sort_order {+ GGML_SORT_ORDER_ASC,+ GGML_SORT_ORDER_DESC,+ };++ GGML_API struct ggml_tensor * ggml_argsort(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_sort_order order);++ // similar to ggml_top_k but implemented as `argsort` + `view`+ GGML_API struct ggml_tensor * ggml_argsort_top_k(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int k);++ // top k elements per row+ // note: the resulting top k indices are in no particular order+ GGML_API struct ggml_tensor * ggml_top_k(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int k);++ GGML_API struct ggml_tensor * ggml_arange(+ struct ggml_context * ctx,+ float start,+ float stop,+ float step);++ // q: [n_embd_k, n_batch, n_head, ne3 ]+ // k: [n_embd_k, n_kv, n_head_kv, ne3 ]+ // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !!+ // mask: [n_kv, n_batch, ne32, ne33]+ // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !!+ //+ // broadcast:+ // n_head % n_head_kv == 0+ // n_head % ne32 == 0+ // ne3 % ne33 == 0+ //+ GGML_API struct ggml_tensor * ggml_flash_attn_ext(+ struct ggml_context * ctx,+ struct ggml_tensor * q,+ struct ggml_tensor * k,+ struct ggml_tensor * v,+ struct ggml_tensor * mask,+ float scale,+ float max_bias,+ float logit_softcap);++ GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(+ struct ggml_tensor * a,+ enum ggml_prec prec),+ "use ggml_prec_set_acc() instead");++ GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(+ const struct ggml_tensor * a);++ // Use finite mask entries as a sparse K/V set. Set 0 to disable.+ // n_kv_max must bound the number of finite entries in every mask row.+ GGML_API void ggml_flash_attn_ext_set_n_kv_max(+ struct ggml_tensor * a,+ int32_t n_kv_max);++ GGML_API void ggml_flash_attn_ext_add_sinks(+ struct ggml_tensor * a,+ struct ggml_tensor * sinks);++ // TODO: needs to be adapted to ggml_flash_attn_ext+ GGML_API struct ggml_tensor * ggml_flash_attn_back(+ struct ggml_context * ctx,+ struct ggml_tensor * q,+ struct ggml_tensor * k,+ struct ggml_tensor * v,+ struct ggml_tensor * d,+ bool masked);++ GGML_API struct ggml_tensor * ggml_ssm_conv(+ struct ggml_context * ctx,+ struct ggml_tensor * sx,+ struct ggml_tensor * c);++ GGML_API struct ggml_tensor * ggml_ssm_scan(+ struct ggml_context * ctx,+ struct ggml_tensor * s,+ struct ggml_tensor * x,+ struct ggml_tensor * dt,+ struct ggml_tensor * A,+ struct ggml_tensor * B,+ struct ggml_tensor * C,+ struct ggml_tensor * ids,+ int64_t K);++ // partition into non-overlapping windows with padding if needed+ // example:+ // a: 768 64 64 1+ // w: 14+ // res: 768 14 14 25+ // used in sam+ GGML_API struct ggml_tensor * ggml_win_part(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int w);++ // reverse of ggml_win_part+ // used in sam+ GGML_API struct ggml_tensor * ggml_win_unpart(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int w0,+ int h0,+ int w);++ GGML_API struct ggml_tensor * ggml_unary(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_unary_op op);++ GGML_API struct ggml_tensor * ggml_unary_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ enum ggml_unary_op op);++ // used in sam+ GGML_API struct ggml_tensor * ggml_get_rel_pos(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ int qh,+ int kh);++ // used in sam+ GGML_API struct ggml_tensor * ggml_add_rel_pos(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * pw,+ struct ggml_tensor * ph);++ GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * pw,+ struct ggml_tensor * ph);++ GGML_API struct ggml_tensor * ggml_rwkv_wkv6(+ struct ggml_context * ctx,+ struct ggml_tensor * k,+ struct ggml_tensor * v,+ struct ggml_tensor * r,+ struct ggml_tensor * tf,+ struct ggml_tensor * td,+ struct ggml_tensor * state);++ GGML_API struct ggml_tensor * ggml_gated_linear_attn(+ struct ggml_context * ctx,+ struct ggml_tensor * k,+ struct ggml_tensor * v,+ struct ggml_tensor * q,+ struct ggml_tensor * g,+ struct ggml_tensor * state,+ float scale);++ GGML_API struct ggml_tensor * ggml_rwkv_wkv7(+ struct ggml_context * ctx,+ struct ggml_tensor * r,+ struct ggml_tensor * w,+ struct ggml_tensor * k,+ struct ggml_tensor * v,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * state);++ /* Solves a specific equation of the form Ax=B, where A is a triangular matrix+ * without zeroes on the diagonal (i.e. invertible).+ * B can have any number of columns, but must have the same number of rows as A+ * If A is [n, n] and B is [n, m], then the result will be [n, m] as well+ * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases+ * where n > 100 sparingly, pre-chunk if necessary.+ *+ * If left = false, solves xA=B instead+ * If lower = false, assumes upper triangular instead+ * If uni = true, assumes diagonal of A to be all ones (will override actual values)+ *+ * TODO: currently only lower, right, non-unitriangular variant is implemented+ */+ GGML_API struct ggml_tensor * ggml_solve_tri(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ bool left,+ bool lower,+ bool uni);++ // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST]+ // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306+ //+ // tensor shapes (S_k == S_v, H_v % H_k == 0):+ // q, k : [S_k, H_k, n_tokens, n_seqs]+ // v : [S_v, H_v, n_tokens, n_seqs]+ // g : [1, H_v, n_tokens, n_seqs] (scalar gate) or [S_v, H_v, n_tokens, n_seqs] (KDA)+ // beta : [1, H_v, n_tokens, n_seqs]+ // state : [S_v, S_v, H_v, n_seqs] -- initial recurrent state s0+ //+ // the output packs the attention scores [S_v, H_v, n_tokens, n_seqs] followed by K state+ // snapshots, most-recent first (slot 0 = final state, slot s = state s tokens back). K == 1+ // keeps only the final state; when n_tokens < K only slots 0..n_tokens-1 are written.+ GGML_API struct ggml_tensor * ggml_gated_delta_net(+ struct ggml_context * ctx,+ struct ggml_tensor * q,+ struct ggml_tensor * k,+ struct ggml_tensor * v,+ struct ggml_tensor * g,+ struct ggml_tensor * beta,+ struct ggml_tensor * state,+ int64_t K);++ // DSA lightning indexer+ //+ // q: [n_embd_idx, n_head_idx, n_batch, ne3 ]+ // k: [n_embd_idx, 1, n_kv, ne3 ]+ // weights: [n_head_idx, n_batch, 1, ne3 ] !! prescaled !!+ // mask: [n_kv, n_batch, 1, ne33] !! f16 !!+ // res: [n_kv, n_batch, 1, ne3 ]+ //+ // broadcast:+ // ne3 % ne33 == 0+ //+ GGML_API struct ggml_tensor * ggml_lightning_indexer(+ struct ggml_context * ctx,+ struct ggml_tensor * q,+ struct ggml_tensor * k,+ struct ggml_tensor * weights,+ struct ggml_tensor * mask);++ // DeepSeek V4 hyper-connections (ref. https://arxiv.org/pdf/2512.24880)+ // In short these operations are replacements for the original residual connection (x = transformer(x) + x)+ // using a richer representation through streams.+ //+ // hc_comb: mixes [(2 + hc)*hc, n_tokens], scale [3], base [(2 + hc)*hc]+ // -> [dst_hc, src_hc, n_tokens]+ // logits[dst, src, t] = mixes[2*hc + dst + hc*src, t]*scale[2]+ // + base[2*hc + dst + hc*src]+ // Softmax over dst, add eps, normalize over src, then repeat normalization+ // over dst followed by src for iterations 1 through n_iter - 1.+ GGML_API struct ggml_tensor * ggml_dsv4_hc_comb(+ struct ggml_context * ctx,+ struct ggml_tensor * mixes,+ struct ggml_tensor * scale,+ struct ggml_tensor * base,+ float eps,+ int32_t n_iter);++ // hc_pre: x [n_embd, hc, n_tokens], weights [hc, n_tokens] -> [n_embd, n_tokens]+ // result[i, t] = sum_h x[i, h, t]*weights[h, t]+ //+ GGML_API struct ggml_tensor * ggml_dsv4_hc_pre(+ struct ggml_context * ctx,+ struct ggml_tensor * x,+ struct ggml_tensor * weights);++ // hc_pre with a per-element gate (Qwen3.8-Flash-Next): gate [n_embd, hc, n_tokens]+ // result[i, t] = scale*sum_h x[i, h, t]*sigmoid(gate[i, h, t])+ //+ GGML_API struct ggml_tensor * ggml_dsv4_hc_pre_gated(+ struct ggml_context * ctx,+ struct ggml_tensor * x,+ struct ggml_tensor * gate,+ float scale);++ // hc_post: x [n_embd, n_tokens], residual [n_embd, hc, n_tokens],+ // post [hc, n_tokens], comb [dst_hc, src_hc, n_tokens]+ // -> [n_embd, hc, n_tokens]+ // result[i, dst, t] = x[i, t]*post[dst, t]+ // + sum_src residual[i, src, t]*comb[dst, src, t]+ // comb == NULL uses the identity: result[i, dst, t] = x[i, t]*post[dst, t] + residual[i, dst, t]+ //+ GGML_API struct ggml_tensor * ggml_dsv4_hc_post(+ struct ggml_context * ctx,+ struct ggml_tensor * x,+ struct ggml_tensor * residual,+ struct ggml_tensor * post,+ struct ggml_tensor * comb);++ // custom operators++ typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);+ typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata);+ typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata);++#define GGML_N_TASKS_MAX (-1)+ // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks++ GGML_API struct ggml_tensor * ggml_map_custom1(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ ggml_custom1_op_t fun,+ int n_tasks,+ void * userdata);++ GGML_API struct ggml_tensor * ggml_map_custom1_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ ggml_custom1_op_t fun,+ int n_tasks,+ void * userdata);++ GGML_API struct ggml_tensor * ggml_map_custom2(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ ggml_custom2_op_t fun,+ int n_tasks,+ void * userdata);++ GGML_API struct ggml_tensor * ggml_map_custom2_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ ggml_custom2_op_t fun,+ int n_tasks,+ void * userdata);++ GGML_API struct ggml_tensor * ggml_map_custom3(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * c,+ ggml_custom3_op_t fun,+ int n_tasks,+ void * userdata);++ GGML_API struct ggml_tensor * ggml_map_custom3_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * b,+ struct ggml_tensor * c,+ ggml_custom3_op_t fun,+ int n_tasks,+ void * userdata);++ typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata);++ GGML_API struct ggml_tensor * ggml_custom_4d(+ struct ggml_context * ctx,+ enum ggml_type type,+ int64_t ne0,+ int64_t ne1,+ int64_t ne2,+ int64_t ne3,+ struct ggml_tensor ** args,+ int n_args,+ ggml_custom_op_t fun,+ int n_tasks,+ void * userdata);++ GGML_API struct ggml_tensor * ggml_custom_inplace(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor ** args,+ int n_args,+ ggml_custom_op_t fun,+ int n_tasks,+ void * userdata);++ // loss function++ GGML_API struct ggml_tensor * ggml_cross_entropy_loss(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // logits+ struct ggml_tensor * b); // labels++ GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back(+ struct ggml_context * ctx,+ struct ggml_tensor * a, // logits+ struct ggml_tensor * b, // labels+ struct ggml_tensor * c); // gradients of cross_entropy_loss result++ // AdamW optimizer step+ // Paper: https://arxiv.org/pdf/1711.05101v3.pdf+ // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html+ GGML_API struct ggml_tensor * ggml_opt_step_adamw(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * grad,+ struct ggml_tensor * m,+ struct ggml_tensor * v,+ struct ggml_tensor * adamw_params); // parameters such as the learning rate++ // stochastic gradient descent step (with weight decay)+ GGML_API struct ggml_tensor * ggml_opt_step_sgd(+ struct ggml_context * ctx,+ struct ggml_tensor * a,+ struct ggml_tensor * grad,+ struct ggml_tensor * sgd_params); // alpha, weight decay++ // build forward multiple tensors and select one of them for computing+ // this is useful for creating graphs that have constant topology but compute different things based on the input+ // ref: https://github.com/ggml-org/llama.cpp/pull/18550+ //+ // nodes:+ // | - build forward into the graph but do not compute+ // c - build forward into the graph and compute+ //+ // | | ... c ... |+ // | | ... c ... |+ // | | ... c ... |+ // [0 1 ... idx ... n-1] <-- ggml_build_forward_select(..., n, idx)+ // c+ // c+ //+ // example:+ // struct ggml_tensor * curs[3];+ //+ // curs[0] = compute0(...);+ // curs[1] = compute1(...);+ // curs[2] = compute2(...);+ //+ // int idx = select_branch(some_input);+ //+ // struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx);+ //+ GGML_API struct ggml_tensor * ggml_build_forward_select(+ struct ggml_cgraph * cgraph,+ struct ggml_tensor ** tensors,+ int n_tensors,+ int idx);++ GGML_API void ggml_build_forward_expand(+ struct ggml_cgraph * cgraph,+ struct ggml_tensor * tensor);++ // add the tensor and its parents to the graph without marking them for compute+ // the flag is set later, when the tensor is reached from a node that computes+ GGML_API void ggml_build_forward_order(+ struct ggml_cgraph * cgraph,+ struct ggml_tensor * tensor);++ GGML_API void ggml_build_backward_expand(+ struct ggml_context * ctx, // context for gradient computation+ struct ggml_cgraph * cgraph,+ struct ggml_tensor ** grad_accs);++ // graph allocation in a context+ GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false+ GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads);+ GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads);+ GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst);+ GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1+ GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph);++ GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph);+ GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i]+ GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph);+ GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph);++ GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);++ GGML_API size_t ggml_graph_overhead(void);+ GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads);++ GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name);+ GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);+ GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node);++ // print info and performance information for the graph+ GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph);++ // dump the graph into a file using the dot format+ GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename);++ // TODO these functions were sandwiched in the old optimization interface, is there a better place for them?+ typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data);++ // Set callback for all future logging events.+ // If this is not called, or NULL is supplied, everything is output on stderr.+ GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data);+ GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data);++ GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);++ //+ // quantization+ //++ // - ggml_quantize_init can be called multiple times with the same type+ // it will only initialize the quantization tables for the first call or after ggml_quantize_free+ // automatically called by ggml_quantize_chunk for convenience+ //+ // - ggml_quantize_free will free any memory allocated by ggml_quantize_init+ // call this at the end of the program to avoid memory leaks+ //+ // note: these are thread-safe+ //+ GGML_API void ggml_quantize_init(enum ggml_type type);+ GGML_API void ggml_quantize_free(void);++ // some quantization type cannot be used without an importance matrix+ GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type);++ // calls ggml_quantize_init internally (i.e. can allocate memory)+ GGML_API size_t ggml_quantize_chunk(+ enum ggml_type type,+ const float * src,+ void * dst,+ int64_t start,+ int64_t nrows,+ int64_t n_per_row,+ const float * imatrix);++#ifdef __cplusplus+ // restrict not standard in C+++# if defined(__GNUC__)+# define GGML_RESTRICT __restrict__+# elif defined(__clang__)+# define GGML_RESTRICT __restrict+# elif defined(_MSC_VER)+# define GGML_RESTRICT __restrict+# else+# define GGML_RESTRICT+# endif+#else+# if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L)+# define GGML_RESTRICT __restrict+# else+# define GGML_RESTRICT restrict+# endif+#endif+ typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);+ typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);++ struct ggml_type_traits {+ const char * type_name;+ int64_t blck_size;+ int64_t blck_size_interleave; // interleave elements in blocks+ size_t type_size;+ bool is_quantized;+ ggml_to_float_t to_float;+ ggml_from_float_t from_float_ref;+ };++ GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);++ // ggml threadpool+ // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend+ // the goal should be to create an API that other backends can use move everything to the ggml base++ // scheduling priorities+ enum ggml_sched_priority {+ GGML_SCHED_PRIO_LOW = -1,+ GGML_SCHED_PRIO_NORMAL,+ GGML_SCHED_PRIO_MEDIUM,+ GGML_SCHED_PRIO_HIGH,+ GGML_SCHED_PRIO_REALTIME+ };++ // threadpool params+ // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults+ struct ggml_threadpool_params {+ bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)+ int n_threads; // number of threads+ enum ggml_sched_priority prio; // thread priority+ uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling)+ bool strict_cpu; // strict cpu placement+ bool paused; // start in paused state+ };++ struct ggml_threadpool; // forward declaration, see ggml.c++ typedef struct ggml_threadpool * ggml_threadpool_t;++ GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);+ GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads);+ GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/include/gguf.h b/python/freetoken/kernel/csrc/ggml/include/gguf.hnew file mode 100644index 00000000..b3a1e123--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/include/gguf.h@@ -0,0 +1,211 @@+// This file contains functionality related to "GGUF" files, the binary file format used by ggml.+// GGUF files have the following structure:+//+// 1. File magic "GGUF" (4 bytes).+// 2. File version (uint32_t).+// 3. Number of ggml tensors in file (int64_t).+// 4. Number of key-value-pairs in file (int64_t).+// 5. For each KV pair:+// 1. The key (string).+// 2. The value type (gguf_type).+// 3a. If the value type is GGUF_TYPE_ARRAY:+// 1. The type of the array (gguf_type).+// 2. The number of elements in the array (uint64_t).+// 3. The binary representation of each element in the array.+// 3b. Otherwise:+// 1. The binary representation of the value.+// 6. For each ggml tensor:+// 1. The tensor name (string).+// 2. The number of dimensions of the tensor (uint32_t).+// 3. For each dimension:+// 1. The size of the tensor in the dimension (int64_t).+// 4. The tensor data type (ggml_type).+// 5. The tensor data offset in the tensor data binary blob (uint64_t).+// 7. The tensor data binary blob (optional, aligned).+//+// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator.+// All enums are stored as int32_t.+// All bool values are stored as int8_t.+// If the special key "general.alignment" (uint32_t) is defined it is used for alignment,+// otherwise GGUF_DEFAULT_ALIGNMENT is used.+//+// Module maintainer: Johannes Gäßler (@JohannesGaessler, [email protected])++#pragma once++#include "ggml.h"++#include <stdbool.h>+#include <stdint.h>++#define GGUF_MAGIC "GGUF"+#define GGUF_VERSION 3++#define GGUF_KEY_GENERAL_ALIGNMENT "general.alignment"++#define GGUF_DEFAULT_ALIGNMENT 32++#ifdef __cplusplus+extern "C" {+#endif++ // types that can be stored as GGUF KV data+ enum gguf_type {+ GGUF_TYPE_UINT8 = 0,+ GGUF_TYPE_INT8 = 1,+ GGUF_TYPE_UINT16 = 2,+ GGUF_TYPE_INT16 = 3,+ GGUF_TYPE_UINT32 = 4,+ GGUF_TYPE_INT32 = 5,+ GGUF_TYPE_FLOAT32 = 6,+ GGUF_TYPE_BOOL = 7,+ GGUF_TYPE_STRING = 8,+ GGUF_TYPE_ARRAY = 9,+ GGUF_TYPE_UINT64 = 10,+ GGUF_TYPE_INT64 = 11,+ GGUF_TYPE_FLOAT64 = 12,+ GGUF_TYPE_COUNT, // marks the end of the enum+ };++ struct gguf_context;++ struct gguf_init_params {+ bool no_alloc;++ // if not NULL, create a ggml_context and allocate the tensor data in it+ struct ggml_context ** ctx;+ };++ // callback to simulate or wrap a FILE pointer - read up to `len` bytes at `offset` into `output` and return the number of bytes read+ typedef size_t (*gguf_reader_callback_t)(void * userdata, void * output, uint64_t offset, size_t len);++ GGML_API struct gguf_context * gguf_init_empty(void);+ GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params);+ GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);+ GGML_API struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, struct gguf_init_params params);++ // max_chunk_read is the maximum number of bytes that the GGUF code will read at once from the callback, a value of 0 means no limit+ GGML_API struct gguf_context * gguf_init_from_callback(gguf_reader_callback_t callback, void * userdata, size_t max_chunk_read, uint64_t max_expected_size, struct gguf_init_params params);++ GGML_API void gguf_free(struct gguf_context * ctx);++ GGML_API const char * gguf_type_name(enum gguf_type type);++ GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx);+ GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx);+ GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); // padded to gguf_get_alignment if and only if the gguf_context contains at least one tensor++ GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx);+ GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found+ GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id);++ GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id);+ GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id);++ // will abort if the wrong type is used for the key+ GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id);+ GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id);+ GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id);+ GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id);+ GGML_API size_t gguf_get_arr_n (const struct gguf_context * ctx, int64_t key_id);++ // get raw pointer to the first element of the array with the given key_id+ // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference)+ GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id);++ // get ith C string from array with given key_id+ GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i);++ GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx);+ GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found+ GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id);+ GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id);+ GGML_API const int64_t * gguf_get_tensor_ne (const struct gguf_context * ctx, int64_t tensor_id); // returns ne, an array of GGML_MAX_DIMS elements; ne[dim] is 1 for dim >= n_dims+ GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id);+ GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id);++ // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist)+ GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key);++ // overrides an existing KV pair or adds a new one, the new KV pair is always at the back+ GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);+ GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);+ GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);+ GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);+ GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);+ GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);+ GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);+ GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val);+ GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val);+ GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val);+ GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);+ GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);++ // creates a new array with n elements of the given type and copies the corresponding number of bytes from data+ GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n);++ // creates a new array with n strings and copies the corresponding strings from data+ GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n);++ // set or add KV pairs from another context+ GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src);++ // add tensor to GGUF context, tensor name must be unique+ GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor);++ // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated+ // in such a way that the tensor data remains as one contiguous block (except for padding)+ GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);++ // assumes that at least gguf_get_tensor_size bytes can be read from data+ GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data);++ // writing gguf files can be done in 3 ways:+ //+ // - write the entire gguf_context to a binary file in a single pass:+ //+ // gguf_write_to_file(ctx, fname, /*only_meta =*/ false);+ //+ // - write only the meta data to a file, then re-open the file and append the tensor data:+ //+ // gguf_write_to_file(ctx, fname, /*only_meta =*/ true);+ // FILE * f = fopen(fname, "ab");+ // fwrite(f, ...); // write tensor data+ // fclose(f);+ //+ // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:+ //+ // FILE * f = fopen(fname, "wb");+ // const size_t size_meta = gguf_get_meta_size(ctx);+ // fseek(f, size_meta, SEEK_SET);+ // fwrite(f, ...); // write tensor data+ // void * data = malloc(size_meta);+ // gguf_get_meta_data(ctx, data);+ // rewind(f);+ // fwrite(data, 1, data, f);+ // free(data);+ // fclose(f);+ //++ // write the entire context to a binary file+ GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta);+ GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta);++ // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding+ GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx);++ // writes the meta data to pointer "data"+ GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data);++#ifdef __cplusplus+}+#endifdiff --git a/python/freetoken/kernel/csrc/ggml/src/ggml-common.h b/python/freetoken/kernel/csrc/ggml/src/ggml-common.hnew file mode 100644index 00000000..1dbbe326--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/src/ggml-common.h@@ -0,0 +1,1911 @@+#ifndef GGML_COMMON_DECL++#if defined(GGML_COMMON_DECL_C)+#include <stdint.h>++typedef uint16_t ggml_half;+typedef uint32_t ggml_half2;++#define GGML_COMMON_AGGR_U+#define GGML_COMMON_AGGR_S++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_CPP)+#include <cstdint>++typedef uint16_t ggml_half;+typedef uint32_t ggml_half2;++// std-c++ allow anonymous unions but some compiler warn on it+#define GGML_COMMON_AGGR_U data+// std-c++ do not allow it.+#define GGML_COMMON_AGGR_S data++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_METAL)+#include <metal_stdlib>++typedef half ggml_half;+typedef half2 ggml_half2;++#define GGML_COMMON_AGGR_U+#define GGML_COMMON_AGGR_S++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_CUDA)+#if defined(GGML_COMMON_DECL_MUSA)+#include <musa_fp16.h>+#else+#include <cuda_fp16.h>+#endif+#include <cstdint>++typedef half ggml_half;+typedef half2 ggml_half2;++#define GGML_COMMON_AGGR_U+#define GGML_COMMON_AGGR_S data++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_HIP)+#include <hip/hip_fp16.h>+#include <cstdint>++typedef half ggml_half;+typedef half2 ggml_half2;++#define GGML_COMMON_AGGR_U+#define GGML_COMMON_AGGR_S data++#define GGML_COMMON_DECL+#elif defined(GGML_COMMON_DECL_SYCL)+#include <sycl/half_type.hpp>+#include <cstdint>++typedef sycl::half ggml_half;+typedef sycl::half2 ggml_half2;++#define GGML_COMMON_AGGR_U+#define GGML_COMMON_AGGR_S data++#define GGML_COMMON_DECL+#endif++#if defined(GGML_COMMON_DECL)++#ifndef __cplusplus+#ifndef static_assert+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)+#define static_assert(cond, msg) _Static_assert(cond, msg)+#else+#define static_assert(cond, msg) struct global_scope_noop_trick+#endif+#endif+#endif // __cplusplus++// QK = number of values after dequantization+// QK_K = super-block size++#define QK_K 256+#define K_SCALE_SIZE 12++#if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL)+// QR = QK / number of values before dequantization+// QI = number of 32 bit integers before dequantization++#define QI1_0 (QK1_0 / 32)+#define QR1_0 1++#define QI2_0 (QK2_0 / 32)+#define QR2_0 1+++#define QI4_0 (QK4_0 / (4 * QR4_0))+#define QR4_0 2++#define QI4_1 (QK4_1 / (4 * QR4_1))+#define QR4_1 2++#define QI_MXFP4 (QK_MXFP4 / (4 * QR_MXFP4))+#define QR_MXFP4 2++#define QI_NVFP4 (QK_NVFP4 / (4 * QR_NVFP4))+#define QR_NVFP4 2++#define QI5_0 (QK5_0 / (4 * QR5_0))+#define QR5_0 2++#define QI5_1 (QK5_1 / (4 * QR5_1))+#define QR5_1 2++#define QI8_0 (QK8_0 / (4 * QR8_0))+#define QR8_0 1++#define QI8_1 (QK8_1 / (4 * QR8_1))+#define QR8_1 1++#define QI2_K (QK_K / (4*QR2_K))+#define QR2_K 4++#define QI3_K (QK_K / (4*QR3_K))+#define QR3_K 4++#define QI4_K (QK_K / (4*QR4_K))+#define QR4_K 2++#define QI5_K (QK_K / (4*QR5_K))+#define QR5_K 2++#define QI6_K (QK_K / (4*QR6_K))+#define QR6_K 2++#define QI2_XXS (QK_K / (4*QR2_XXS))+#define QR2_XXS 4++#define QI2_XS (QK_K / (4*QR2_XS))+#define QR2_XS 4++#define QI2_S (QK_K / (4*QR2_S))+#define QR2_S 4++#define QI3_XXS (QK_K / (4*QR3_XXS))+#define QR3_XXS 4++#define QI3_XS (QK_K / (4*QR3_XS))+#define QR3_XS 4++#define QI1_S (QK_K / (4*QR1_S))+#define QR1_S 8++#define QI1_M (QK_K / (4*QR1_M))+#define QR1_M 8++#define QI4_NL (QK4_NL / (4*QR4_NL))+#define QR4_NL 2++#define QI4_XS (QK_K / (4*QR4_XS))+#define QR4_XS 2++#define QI3_S (QK_K / (4*QR3_S))+#define QR3_S 4++#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP++#ifdef _MSC_VER+#define GGML_EXTENSION+#else // _MSC_VER+#define GGML_EXTENSION __extension__+#endif // _MSC_VER++#define QK1_0 128+typedef struct {+ ggml_half d; // delta+ uint8_t qs[QK1_0 / 8]; // bits / quants+} block_q1_0;+static_assert(sizeof(block_q1_0) == sizeof(ggml_half) + QK1_0 / 8, "wrong q1_0 block size/padding");++#define QK2_0 64+typedef struct {+ ggml_half d; // delta (scale)+ uint8_t qs[QK2_0 / 4]; // 2 bits per element+} block_q2_0;+static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding");++#define QK4_0 32+typedef struct {+ ggml_half d; // delta+ uint8_t qs[QK4_0 / 2]; // nibbles / quants+} block_q4_0;+static_assert(sizeof(block_q4_0) == sizeof(ggml_half) + QK4_0 / 2, "wrong q4_0 block size/padding");++#define QK4_1 32+typedef struct {+ GGML_EXTENSION union {+ struct {+ ggml_half d; // delta+ ggml_half m; // min+ } GGML_COMMON_AGGR_S;+ ggml_half2 dm;+ } GGML_COMMON_AGGR_U;+ uint8_t qs[QK4_1 / 2]; // nibbles / quants+} block_q4_1;+static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_half) + QK4_1 / 2, "wrong q4_1 block size/padding");++#define QK_MXFP4 32+typedef struct {+ uint8_t e; // E8M0+ uint8_t qs[QK_MXFP4/2];+} block_mxfp4;+static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + QK_MXFP4/2, "wrong mxfp4 block size/padding");++#define QK_NVFP4 64+#define QK_NVFP4_SUB 16 // sub-block size for per-group scales+typedef struct {+ uint8_t d[QK_NVFP4/QK_NVFP4_SUB]; // UE4M3 scales (4 bytes, one per 16-element sub-block)+ uint8_t qs[QK_NVFP4/2]; // packed 4-bit E2M1 values (32 bytes)+} block_nvfp4;+static_assert(sizeof(block_nvfp4) == sizeof(uint8_t)*(QK_NVFP4/QK_NVFP4_SUB) + QK_NVFP4/2, "wrong nvfp4 block size/padding");++#define QK5_0 32+typedef struct {+ ggml_half d; // delta+ uint8_t qh[4]; // 5-th bit of quants+ uint8_t qs[QK5_0 / 2]; // nibbles / quants+} block_q5_0;+static_assert(sizeof(block_q5_0) == sizeof(ggml_half) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding");++#define QK5_1 32+typedef struct {+ GGML_EXTENSION union {+ struct {+ ggml_half d; // delta+ ggml_half m; // min+ } GGML_COMMON_AGGR_S;+ ggml_half2 dm;+ } GGML_COMMON_AGGR_U;+ uint8_t qh[4]; // 5-th bit of quants+ uint8_t qs[QK5_1 / 2]; // nibbles / quants+} block_q5_1;+static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_half) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding");++#define QK8_0 32+typedef struct {+ ggml_half d; // delta+ int8_t qs[QK8_0]; // quants+} block_q8_0;+static_assert(sizeof(block_q8_0) == sizeof(ggml_half) + QK8_0, "wrong q8_0 block size/padding");++#define QK8_1 32+typedef struct {+ GGML_EXTENSION union {+ struct {+ ggml_half d; // delta+ ggml_half s; // d * sum(qs[i])+ } GGML_COMMON_AGGR_S;+ ggml_half2 ds;+ } GGML_COMMON_AGGR_U;+ int8_t qs[QK8_1]; // quants+} block_q8_1;+static_assert(sizeof(block_q8_1) == 2*sizeof(ggml_half) + QK8_1, "wrong q8_1 block size/padding");++//+// Ternary quantization+//++// 1.6875 bpw+typedef struct {+ uint8_t qs[(QK_K - 4 * QK_K / 64) / 5]; // 5 elements per byte (3^5 = 243 < 256)+ uint8_t qh[QK_K/64]; // 4 elements per byte+ ggml_half d;+} block_tq1_0;+static_assert(sizeof(block_tq1_0) == sizeof(ggml_half) + QK_K / 64 + (QK_K - 4 * QK_K / 64) / 5, "wrong tq1_0 block size/padding");++// 2.0625 bpw+typedef struct {+ uint8_t qs[QK_K/4]; // 2 bits per element+ ggml_half d;+} block_tq2_0;+static_assert(sizeof(block_tq2_0) == sizeof(ggml_half) + QK_K / 4, "wrong tq2_0 block size/padding");++//+// Super-block quantization structures+//++// 2-bit quantization+// weight is represented as x = a * q + b+// 16 blocks of 16 elements each+// Effectively 2.625 bits per weight+typedef struct {+ uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits+ uint8_t qs[QK_K/4]; // quants+ GGML_EXTENSION union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR_S;+ ggml_half2 dm;+ } GGML_COMMON_AGGR_U;+} block_q2_K;+static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding");++// 3-bit quantization+// weight is represented as x = a * q+// 16 blocks of 16 elements each+// Effectively 3.4375 bits per weight+typedef struct {+ uint8_t hmask[QK_K/8]; // quants - high bit+ uint8_t qs[QK_K/4]; // quants - low 2 bits+ uint8_t scales[12]; // scales, quantized with 6 bits+ ggml_half d; // super-block scale+} block_q3_K;+static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding");++// 4-bit quantization+// 8 blocks of 32 elements each+// weight is represented as x = a * q + b+// Effectively 4.5 bits per weight+typedef struct {+ GGML_EXTENSION union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR_S;+ ggml_half2 dm;+ } GGML_COMMON_AGGR_U;+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits+ uint8_t qs[QK_K/2]; // 4--bit quants+} block_q4_K;+static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding");++// 5-bit quantization+// 8 blocks of 32 elements each+// weight is represented as x = a * q + b+// Effectively 5.5 bits per weight+typedef struct {+ GGML_EXTENSION union {+ struct {+ ggml_half d; // super-block scale for quantized scales+ ggml_half dmin; // super-block scale for quantized mins+ } GGML_COMMON_AGGR_S;+ ggml_half2 dm;+ } GGML_COMMON_AGGR_U;+ uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits+ uint8_t qh[QK_K/8]; // quants, high bit+ uint8_t qs[QK_K/2]; // quants, low 4 bits+} block_q5_K;+static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding");++// 6-bit quantization+// weight is represented as x = a * q+// 16 blocks of 16 elements each+// Effectively 6.5625 bits per weight+typedef struct {+ uint8_t ql[QK_K/2]; // quants, lower 4 bits+ uint8_t qh[QK_K/4]; // quants, upper 2 bits+ int8_t scales[QK_K/16]; // scales, quantized with 8 bits+ ggml_half d; // super-block scale+} block_q6_K;+static_assert(sizeof(block_q6_K) == sizeof(ggml_half) + QK_K / 16 + 3*QK_K/4, "wrong q6_K block size/padding");++// This is only used for intermediate quantization and dot products+typedef struct {+ float d; // delta+ int8_t qs[QK_K]; // quants+ int16_t bsums[QK_K/16]; // sum of quants in groups of 16+} block_q8_K;+static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_t), "wrong q8_K block size/padding");++// (Almost) "true" 2-bit quantization.+// Due to the need to use blocks as per ggml design, it ends up using+// 2.0625 bpw because of the 16-bit scale for each block of 256.+typedef struct {+ ggml_half d;+ uint16_t qs[QK_K/8];+} block_iq2_xxs;+static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");++// 2.3125 bpw quants+typedef struct {+ ggml_half d;+ uint16_t qs[QK_K/8];+ uint8_t scales[QK_K/32];+} block_iq2_xs;+static_assert(sizeof(block_iq2_xs) == sizeof(ggml_half) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");++// 2.5625 bpw quants+typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/4];+ uint8_t qh[QK_K/32];+ uint8_t scales[QK_K/32];+} block_iq2_s;+static_assert(sizeof(block_iq2_s) == sizeof(ggml_half) + QK_K/4 + QK_K/16, "wrong iq2_s block size/padding");++// (Almost) "true" 3-bit quantization.+// Due to the need to use blocks as per ggml design, it ends up using+// 3.0625 bpw because of the 16-bit scale for each block of 256.+typedef struct {+ ggml_half d;+ uint8_t qs[3*QK_K/8];+} block_iq3_xxs;+static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), "wrong iq3_xxs block size/padding");++// 3.4375 bpw+#define IQ3S_N_SCALE QK_K/64+typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/4];+ uint8_t qh[QK_K/32];+ uint8_t signs[QK_K/8];+ uint8_t scales[IQ3S_N_SCALE];+} block_iq3_s;+static_assert(sizeof(block_iq3_s) == sizeof(ggml_half) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding");++// 1.5625 bpw+typedef struct {+ ggml_half d;+ uint8_t qs[QK_K/8];+ uint16_t qh[QK_K/32];+} block_iq1_s;+static_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, "wrong iq1_s block size/padding");++// 1.75 bpw+typedef struct {+ uint8_t qs[QK_K/8]; // grid index, low 8 bits+ uint8_t qh[QK_K/16]; // grid index, high 3 bits + grid shift bit (for two groups of 8)+ uint8_t scales[QK_K/32]; // 3-bit block scales (4-bit if QK_K == 64)+} block_iq1_m;+static_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, "wrong iq1_m block size/padding");++// Used by IQ1_M quants+typedef union {+ ggml_half f16;+ uint16_t u16;+} iq1m_scale_t;++// Non-linear quants+#define QK4_NL 32+typedef struct {+ ggml_half d;+ uint8_t qs[QK4_NL/2];+} block_iq4_nl;+static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "wrong iq4_nl block size/padding");++typedef struct {+ ggml_half d;+ uint16_t scales_h;+ uint8_t scales_l[QK_K/64];+ uint8_t qs[QK_K/2];+} block_iq4_xs;+static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding");++#endif // GGML_COMMON_DECL+#endif // GGML_COMMON_DECL++////////////////////////////////////////////////////////////////////////////////++#ifndef GGML_COMMON_IMPL++#if defined(GGML_COMMON_IMPL_C)+#include <stdint.h>++#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {+#define GGML_TABLE_END() };++#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_CPP)+#include <cstdint>++#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {+#define GGML_TABLE_END() };++#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_METAL)+#include <metal_stdlib>++#define GGML_TABLE_BEGIN(type, name, size) static const constant type name[size] = {+#define GGML_TABLE_END() };++#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_CUDA) || defined(GGML_COMMON_IMPL_HIP) || defined(GGML_COMMON_IMPL_MUSA)+#include <cstdint>++#define GGML_TABLE_BEGIN(type, name, size) static const __device__ type name[size] = {+#define GGML_TABLE_END() };++#define GGML_COMMON_IMPL+#elif defined(GGML_COMMON_IMPL_SYCL)++#include <cstdint>++#define GGML_TABLE_BEGIN(type, name, size) static const type name[size] = {+#define GGML_TABLE_END() };++#define GGML_COMMON_IMPL+#endif++#if defined(GGML_COMMON_IMPL)++GGML_TABLE_BEGIN(uint8_t, kmask_iq2xs, 8)+ 1, 2, 4, 8, 16, 32, 64, 128+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint8_t, ksigns_iq2xs, 128)+ 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15,+ 144, 17, 18, 147, 20, 149, 150, 23, 24, 153, 154, 27, 156, 29, 30, 159,+ 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43, 172, 45, 46, 175,+ 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63,+ 192, 65, 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207,+ 80, 209, 210, 83, 212, 85, 86, 215, 216, 89, 90, 219, 92, 221, 222, 95,+ 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237, 238, 111,+ 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint64_t, ksigns64, 128)+ 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff,+ 0xff00000000ff0000, 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff,+ 0xff000000ff000000, 0x00000000ff0000ff, 0x00000000ff00ff00, 0xff000000ff00ffff,+ 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00, 0x00000000ffffffff,+ 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,+ 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff,+ 0x000000ffff000000, 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff,+ 0xff0000ffffff0000, 0x000000ffffff00ff, 0x000000ffffffff00, 0xff0000ffffffffff,+ 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00, 0xff00ff000000ffff,+ 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,+ 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff,+ 0xff00ff00ffff0000, 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff,+ 0x0000ffff00000000, 0xff00ffff000000ff, 0xff00ffff0000ff00, 0x0000ffff0000ffff,+ 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00, 0xff00ffff00ffffff,+ 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,+ 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff,+ 0xffff000000000000, 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff,+ 0x00ff000000ff0000, 0xffff000000ff00ff, 0xffff000000ffff00, 0x00ff000000ffffff,+ 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0x00ff0000ff00ffff,+ 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,+ 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff,+ 0xffff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff,+ 0xffff00ffff000000, 0x00ff00ffff0000ff, 0x00ff00ffff00ff00, 0xffff00ffff00ffff,+ 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00, 0x00ff00ffffffffff,+ 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,+ 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff,+ 0xffffff00ff000000, 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff,+ 0x00ffff00ffff0000, 0xffffff00ffff00ff, 0xffffff00ffffff00, 0x00ffff00ffffffff,+ 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00, 0xffffffff0000ffff,+ 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,+ 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff,+ 0xffffffffffff0000, 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,+GGML_TABLE_END()+++GGML_TABLE_BEGIN(uint64_t, iq2xxs_grid, 256)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808,+ 0x08080808082b082b, 0x08080808082b2b08, 0x08080808082b2b2b, 0x0808080819080819,+ 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08, 0x08080808192b0819,+ 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,+ 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808,+ 0x0808081908191919, 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08,+ 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b082b082b, 0x0808082b2b08082b,+ 0x0808190808080819, 0x0808190808081908, 0x0808190808190808, 0x08081908082b0819,+ 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,+ 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808,+ 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08,+ 0x08081919082b0808, 0x080819191908192b, 0x08081919192b2b19, 0x080819192b080808,+ 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b19080808,+ 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,+ 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08,+ 0x08082b1908081908, 0x08082b1919080808, 0x08082b2b0808082b, 0x08082b2b08191908,+ 0x0819080808080819, 0x0819080808081908, 0x0819080808190808, 0x08190808082b0819,+ 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,+ 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808,+ 0x0819081919190808, 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908,+ 0x0819082b19081919, 0x0819190808080808, 0x0819190808082b08, 0x08191908082b0808,+ 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808, 0x0819191908192b08,+ 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,+ 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819,+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808082b2b, 0x082b080819081908,+ 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b, 0x082b0819082b2b19,+ 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,+ 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b,+ 0x082b191908080808, 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808,+ 0x082b2b0808082b08, 0x082b2b08082b0808, 0x082b2b082b191908, 0x082b2b2b19081908,+ 0x1908080808080819, 0x1908080808081908, 0x1908080808190808, 0x1908080808192b08,+ 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,+ 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908,+ 0x190808082b190808, 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819,+ 0x190808192b080808, 0x190808192b081919, 0x1908082b08080819, 0x1908082b08190808,+ 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08, 0x1908190808080808,+ 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,+ 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819,+ 0x19082b0808081908, 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919,+ 0x19082b1908080808, 0x19082b1919192b08, 0x19082b19192b0819, 0x19082b192b08082b,+ 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808, 0x1919080808082b08,+ 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,+ 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908,+ 0x1919082b2b190819, 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b,+ 0x1919192b08080819, 0x1919192b19191908, 0x19192b0808080808, 0x19192b0808190819,+ 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808, 0x19192b2b08082b08,+ 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,+ 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808,+ 0x192b190808080808, 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b,+ 0x192b19192b081908, 0x192b2b081908082b, 0x2b08080808080808, 0x2b0808080808082b,+ 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b, 0x2b08081908081908,+ 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,+ 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808,+ 0x2b081908192b0808, 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908,+ 0x2b08192b08082b19, 0x2b08192b19080808, 0x2b08192b192b0808, 0x2b082b080808082b,+ 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908, 0x2b19080808190808,+ 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,+ 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b,+ 0x2b19190819081908, 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808,+ 0x2b2b08080808082b, 0x2b2b080819190808, 0x2b2b08082b081919, 0x2b2b081908082b19,+ 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808, 0x2b2b2b1908081908,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint64_t, iq2xs_grid, 512)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808,+ 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,+ 0x080808082b191908, 0x080808082b192b19, 0x080808082b2b0808, 0x0808081908080819,+ 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808,+ 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,+ 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b,+ 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,+ 0x08080819192b0808, 0x08080819192b2b08, 0x080808192b080819, 0x080808192b081908,+ 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,+ 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,+ 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908,+ 0x080819080808192b, 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b,+ 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908,+ 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,+ 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808,+ 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x0808191908080808,+ 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908190819,+ 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819, 0x0808191919081908,+ 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808,+ 0x0808192b1908082b, 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808082b2b, 0x08082b0808190819,+ 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b0819080819,+ 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,+ 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,+ 0x08082b1908190808, 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19,+ 0x08082b2b08080808, 0x08082b2b082b0808, 0x08082b2b082b2b08, 0x08082b2b2b19192b,+ 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b,+ 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,+ 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808,+ 0x081908081908082b, 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819,+ 0x0819080819191908, 0x08190808192b0808, 0x08190808192b2b2b, 0x081908082b080819,+ 0x081908082b081908, 0x081908082b190808, 0x0819081908080808, 0x081908190808082b,+ 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,+ 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808,+ 0x081908192b080808, 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819,+ 0x0819082b08081908, 0x0819082b0808192b, 0x0819082b08190808, 0x0819082b19080808,+ 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b, 0x0819190808081919,+ 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,+ 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808,+ 0x08191908192b1908, 0x081919082b080808, 0x0819191908080819, 0x0819191908081908,+ 0x0819191908190808, 0x0819191919080808, 0x0819192b08080808, 0x0819192b08191908,+ 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908, 0x08192b0808190808,+ 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,+ 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819,+ 0x08192b2b2b2b2b19, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,+ 0x082b080808082b08, 0x082b080808082b2b, 0x082b080808190819, 0x082b080808191908,+ 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908, 0x082b080819190808,+ 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,+ 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919,+ 0x082b082b08080808, 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08,+ 0x082b190808080819, 0x082b190808081908, 0x082b190808190808, 0x082b1908082b2b19,+ 0x082b190819080808, 0x082b191908080808, 0x082b191919080819, 0x082b19191919082b,+ 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,+ 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808,+ 0x082b2b0819191919, 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08,+ 0x082b2b192b190808, 0x082b2b2b08082b08, 0x082b2b2b082b0808, 0x082b2b2b2b08082b,+ 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908,+ 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,+ 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908,+ 0x1908080819080808, 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08,+ 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908, 0x19080808192b0808,+ 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,+ 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,+ 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819,+ 0x1908081919081908, 0x1908081919190808, 0x190808192b080808, 0x190808192b081919,+ 0x190808192b2b082b, 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808,+ 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808, 0x1908190808080808,+ 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,+ 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819,+ 0x1908190819081908, 0x1908190819190808, 0x190819082b080808, 0x190819082b191908,+ 0x1908191908080819, 0x1908191908081908, 0x1908191908190808, 0x19081919082b1908,+ 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808, 0x1908192b08082b2b,+ 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,+ 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908,+ 0x19082b08192b082b, 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908,+ 0x19082b1919190808, 0x19082b19192b2b19, 0x19082b2b08081908, 0x1919080808080808,+ 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819,+ 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,+ 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819,+ 0x1919081908081908, 0x1919081908190808, 0x1919081908191919, 0x1919081919080808,+ 0x191908191908082b, 0x1919082b08080808, 0x1919082b19081908, 0x1919082b2b2b2b2b,+ 0x1919190808080819, 0x1919190808081908, 0x1919190808190808, 0x19191908082b0819,+ 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,+ 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08,+ 0x1919192b082b0819, 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808,+ 0x19192b0808191908, 0x19192b0819080819, 0x19192b0819190808, 0x19192b082b192b19,+ 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b, 0x19192b2b2b081919,+ 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,+ 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19,+ 0x192b081908080808, 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b,+ 0x192b190808080808, 0x192b19080819192b, 0x192b191908190808, 0x192b191919080808,+ 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819, 0x192b2b08192b2b2b,+ 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,+ 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08,+ 0x2b08080808190819, 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808082b080808,+ 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,+ 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808,+ 0x2b08082b2b080808, 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08,+ 0x2b08190808080819, 0x2b08190808081908, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808, 0x2b0819082b082b19,+ 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,+ 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919,+ 0x2b082b0819192b2b, 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08,+ 0x2b082b190808192b, 0x2b082b2b082b082b, 0x2b082b2b2b080808, 0x2b082b2b2b082b08,+ 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819, 0x2b19080808081908,+ 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,+ 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b,+ 0x2b19082b2b082b19, 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908,+ 0x2b19190819190808, 0x2b19190819192b08, 0x2b191919082b2b19, 0x2b1919192b190808,+ 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819, 0x2b192b082b2b192b,+ 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,+ 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808,+ 0x2b2b0808082b2b2b, 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19,+ 0x2b2b08192b2b192b, 0x2b2b082b08080808, 0x2b2b082b0808082b, 0x2b2b082b08082b08,+ 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808, 0x2b2b190819080808,+ 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,+ 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808,+ 0x2b2b2b082b2b2b08, 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b,+ 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b, 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint64_t, iq2s_grid, 1024)+ 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08,+ 0x0808080808082b2b, 0x0808080808190819, 0x0808080808191908, 0x080808080819192b,+ 0x0808080808192b19, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b1919,+ 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908, 0x080808081908192b,+ 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,+ 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b,+ 0x08080808192b2b19, 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919,+ 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b2b0808,+ 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819, 0x0808081908081908,+ 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,+ 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908,+ 0x0808081919080808, 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08,+ 0x0808081919190819, 0x0808081919191908, 0x080808191919192b, 0x0808081919192b19,+ 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08, 0x080808192b080819,+ 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,+ 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b,+ 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908,+ 0x0808082b082b0808, 0x0808082b082b2b2b, 0x0808082b19080819, 0x0808082b19081908,+ 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808, 0x0808082b19191919,+ 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,+ 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b,+ 0x0808190808082b19, 0x0808190808190808, 0x080819080819082b, 0x0808190808191919,+ 0x0808190808192b08, 0x08081908082b0819, 0x08081908082b1908, 0x08081908082b192b,+ 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919,+ 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,+ 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b,+ 0x08081908192b1919, 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b,+ 0x080819082b082b19, 0x080819082b190808, 0x080819082b191919, 0x080819082b192b08,+ 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808, 0x080819190808082b,+ 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,+ 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808,+ 0x08081919082b1919, 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908,+ 0x080819191908192b, 0x0808191919082b19, 0x0808191919190808, 0x080819191919082b,+ 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819, 0x08081919192b1908,+ 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,+ 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819,+ 0x0808192b08081908, 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808,+ 0x0808192b08191919, 0x0808192b19080808, 0x0808192b19081919, 0x0808192b19082b08,+ 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808, 0x0808192b2b080819,+ 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,+ 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908,+ 0x08082b080819192b, 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919,+ 0x08082b08082b2b2b, 0x08082b0819080819, 0x08082b0819081908, 0x08082b081908192b,+ 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b, 0x08082b0819191919,+ 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,+ 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819,+ 0x08082b1908081908, 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919,+ 0x08082b1908192b08, 0x08082b19082b0819, 0x08082b1919080808, 0x08082b1919081919,+ 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908, 0x08082b19192b0808,+ 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,+ 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b,+ 0x08082b2b19190808, 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908,+ 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b,+ 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908,+ 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,+ 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b,+ 0x0819080819192b19, 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919,+ 0x08190808192b2b08, 0x081908082b080819, 0x081908082b081908, 0x081908082b08192b,+ 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08, 0x081908082b2b0819,+ 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,+ 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908,+ 0x081908190819192b, 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b,+ 0x08190819082b1919, 0x08190819082b2b08, 0x0819081919080819, 0x0819081919081908,+ 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808, 0x081908191919082b,+ 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,+ 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08,+ 0x081908192b190819, 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908,+ 0x0819082b08082b19, 0x0819082b08190808, 0x0819082b08191919, 0x0819082b082b0819,+ 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919, 0x0819082b19190819,+ 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,+ 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08,+ 0x0819190808190819, 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19,+ 0x08191908082b0808, 0x08191908082b1919, 0x08191908082b2b08, 0x0819190819080819,+ 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19, 0x0819190819190808,+ 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,+ 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919,+ 0x081919082b082b08, 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808,+ 0x0819191908080819, 0x0819191908081908, 0x081919190808192b, 0x0819191908082b19,+ 0x0819191908190808, 0x081919190819082b, 0x0819191908191919, 0x0819191908192b08,+ 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,+ 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908,+ 0x08191919192b0808, 0x081919192b080819, 0x081919192b081908, 0x081919192b190808,+ 0x0819192b08080808, 0x0819192b08081919, 0x0819192b08082b08, 0x0819192b08190819,+ 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819, 0x0819192b19081908,+ 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,+ 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808,+ 0x08192b0808191919, 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808,+ 0x08192b081908082b, 0x08192b0819081919, 0x08192b0819082b08, 0x08192b0819190819,+ 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819, 0x08192b082b081908,+ 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,+ 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819,+ 0x08192b1919081908, 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b,+ 0x08192b2b08081908, 0x08192b2b08190808, 0x08192b2b19080808, 0x08192b2b1919192b,+ 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08,+ 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,+ 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819,+ 0x082b080819081908, 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919,+ 0x082b0808192b1908, 0x082b08082b080808, 0x082b08082b082b2b, 0x082b08082b191908,+ 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908, 0x082b081908190808,+ 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,+ 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908,+ 0x082b0819192b0808, 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808,+ 0x082b082b08080808, 0x082b082b08082b2b, 0x082b082b082b082b, 0x082b082b082b2b08,+ 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808, 0x082b082b2b082b08,+ 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,+ 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919,+ 0x082b190808192b08, 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808,+ 0x082b19081908082b, 0x082b190819081919, 0x082b190819082b08, 0x082b190819190819,+ 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819, 0x082b19082b081908,+ 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,+ 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819,+ 0x082b191919081908, 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808,+ 0x082b192b08080819, 0x082b192b08081908, 0x082b192b08190808, 0x082b192b19080808,+ 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919, 0x082b2b0808190819,+ 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,+ 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908,+ 0x082b2b1908190808, 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b,+ 0x082b2b2b192b1908, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,+ 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808,+ 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,+ 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808,+ 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b,+ 0x1908080819190819, 0x1908080819191908, 0x190808081919192b, 0x1908080819192b19,+ 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919, 0x190808082b080819,+ 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,+ 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b,+ 0x1908081908081919, 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908,+ 0x190808190819192b, 0x1908081908192b19, 0x19080819082b0808, 0x19080819082b082b,+ 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908, 0x190808191908192b,+ 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,+ 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808,+ 0x190808192b08082b, 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819,+ 0x190808192b191908, 0x190808192b2b0808, 0x1908082b08080819, 0x1908082b08081908,+ 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919, 0x1908082b08192b08,+ 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,+ 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819,+ 0x1908082b2b081908, 0x1908190808080808, 0x190819080808082b, 0x1908190808081919,+ 0x1908190808082b08, 0x1908190808082b2b, 0x1908190808190819, 0x1908190808191908,+ 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808, 0x19081908082b082b,+ 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,+ 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b,+ 0x1908190819191919, 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908,+ 0x190819082b080808, 0x190819082b08082b, 0x190819082b081919, 0x190819082b082b08,+ 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808, 0x1908191908080819,+ 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,+ 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819,+ 0x19081919082b1908, 0x1908191919080808, 0x190819191908082b, 0x1908191919081919,+ 0x1908191919082b08, 0x1908191919190819, 0x1908191919191908, 0x19081919192b0808,+ 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908, 0x190819192b190808,+ 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,+ 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819,+ 0x1908192b19081908, 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919,+ 0x19082b0808080819, 0x19082b0808081908, 0x19082b0808082b19, 0x19082b0808190808,+ 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08, 0x19082b08082b0819,+ 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,+ 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808,+ 0x19082b082b081908, 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b,+ 0x19082b1908081919, 0x19082b1908082b08, 0x19082b1908190819, 0x19082b1908191908,+ 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908, 0x19082b1919190808,+ 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,+ 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b,+ 0x1919080808081919, 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908,+ 0x191908080819192b, 0x1919080808192b19, 0x19190808082b0808, 0x19190808082b082b,+ 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819, 0x1919080819081908,+ 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,+ 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908,+ 0x191908082b080808, 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08,+ 0x191908082b190819, 0x191908082b191908, 0x1919081908080819, 0x1919081908081908,+ 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808, 0x191908190819082b,+ 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,+ 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08,+ 0x1919081919190819, 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819,+ 0x191908192b081908, 0x191908192b190808, 0x1919082b08080808, 0x1919082b08081919,+ 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908, 0x1919082b082b0808,+ 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,+ 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b,+ 0x1919190808082b19, 0x1919190808190808, 0x191919080819082b, 0x1919190808191919,+ 0x1919190808192b08, 0x19191908082b0819, 0x19191908082b1908, 0x1919190819080808,+ 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08, 0x1919190819190819,+ 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,+ 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919,+ 0x1919191908082b08, 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808,+ 0x1919191919080819, 0x1919191919081908, 0x1919191919190808, 0x191919192b080808,+ 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808, 0x1919192b082b192b,+ 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,+ 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808,+ 0x19192b0819080819, 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b,+ 0x19192b082b080808, 0x19192b1908080819, 0x19192b1908081908, 0x19192b1908190808,+ 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19, 0x19192b2b2b081919,+ 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,+ 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08,+ 0x192b0808082b0819, 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919,+ 0x192b080819082b08, 0x192b080819190819, 0x192b080819191908, 0x192b0808192b0808,+ 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808, 0x192b08190808082b,+ 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,+ 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808,+ 0x192b08192b080808, 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808,+ 0x192b082b19080808, 0x192b082b1919192b, 0x192b082b2b2b0819, 0x192b190808080808,+ 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819, 0x192b190808191908,+ 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,+ 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808,+ 0x192b191919080808, 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b,+ 0x192b192b08080808, 0x192b192b2b191908, 0x192b2b0808080819, 0x192b2b0808081908,+ 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08, 0x192b2b1908080808,+ 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,+ 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,+ 0x2b08080808191908, 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919,+ 0x2b08080819080819, 0x2b08080819081908, 0x2b08080819190808, 0x2b0808081919082b,+ 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819, 0x2b0808082b080808,+ 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,+ 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b,+ 0x2b08081908191919, 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908,+ 0x2b08081919080808, 0x2b0808191908082b, 0x2b08081919081919, 0x2b08081919082b08,+ 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819, 0x2b0808192b081908,+ 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,+ 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819,+ 0x2b08082b19081908, 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908,+ 0x2b0819080808192b, 0x2b08190808082b19, 0x2b08190808190808, 0x2b0819080819082b,+ 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819, 0x2b08190819080808,+ 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,+ 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908,+ 0x2b0819082b190808, 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919,+ 0x2b08191908082b08, 0x2b08191908190819, 0x2b08191908191908, 0x2b081919082b0808,+ 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808, 0x2b0819192b080808,+ 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,+ 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919,+ 0x2b082b0808190819, 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908,+ 0x2b082b0819190808, 0x2b082b082b2b082b, 0x2b082b1908080819, 0x2b082b1908081908,+ 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b, 0x2b082b2b19192b08,+ 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,+ 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b,+ 0x2b19080808191919, 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808,+ 0x2b1908081908082b, 0x2b19080819081919, 0x2b19080819082b08, 0x2b19080819190819,+ 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819, 0x2b1908082b081908,+ 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,+ 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808,+ 0x2b19081919192b2b, 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808,+ 0x2b19082b19080808, 0x2b19082b2b2b192b, 0x2b19190808080808, 0x2b1919080808082b,+ 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819, 0x2b19190808191908,+ 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,+ 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908,+ 0x2b19191908190808, 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819,+ 0x2b19192b08080808, 0x2b19192b1908192b, 0x2b19192b192b1908, 0x2b192b0808080819,+ 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b, 0x2b192b0819080808,+ 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,+ 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b,+ 0x2b2b080808191908, 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819,+ 0x2b2b080819081908, 0x2b2b080819190808, 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b,+ 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b, 0x2b2b082b08082b2b,+ 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,+ 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819,+ 0x2b2b190808081908, 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19,+ 0x2b2b19082b2b1908, 0x2b2b191908080808, 0x2b2b191908192b19, 0x2b2b192b19190819,+ 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b, 0x2b2b2b1919191908,+ 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,+ 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint32_t, iq3xxs_grid, 256)+ 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414,+ 0x04041c0c, 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14,+ 0x040c140c, 0x040c142c, 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404,+ 0x04140414, 0x04140424, 0x04140c0c, 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e,+ 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c, 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c,+ 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e, 0x04243e1c, 0x04243e2c, 0x042c040c,+ 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04, 0x043e0c24, 0x043e0c34,+ 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c, 0x0c04141c,+ 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,+ 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04,+ 0x0c143e14, 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c,+ 0x0c24042c, 0x0c242c04, 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414,+ 0x0c3e2404, 0x14040404, 0x14040414, 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434,+ 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c, 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c,+ 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404, 0x14140414, 0x14140c0c, 0x14140c3e,+ 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c, 0x141c0c04, 0x141c0c24,+ 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c, 0x142c3e24,+ 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,+ 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c,+ 0x1c0c2424, 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14,+ 0x1c1c0c0c, 0x1c1c1c1c, 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414,+ 0x1c2c2c2c, 0x1c340c24, 0x1c341c34, 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e,+ 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e, 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404,+ 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424, 0x24242c0c, 0x24243424, 0x242c142c,+ 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04, 0x2c040c14, 0x2c04240c,+ 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14, 0x2c143e14,+ 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,+ 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c,+ 0x340c340c, 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14,+ 0x34341c1c, 0x343e041c, 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14,+ 0x3e042c14, 0x3e0c1434, 0x3e0c2404, 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c,+ 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c, 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,+GGML_TABLE_END()++GGML_TABLE_BEGIN(uint32_t, iq3s_grid, 512)+ 0x01010101, 0x01010103, 0x01010105, 0x0101010b, 0x0101010f, 0x01010301, 0x01010303, 0x01010305,+ 0x01010309, 0x0101030d, 0x01010501, 0x01010503, 0x0101050b, 0x01010707, 0x01010901, 0x01010905,+ 0x0101090b, 0x0101090f, 0x01010b03, 0x01010b07, 0x01010d01, 0x01010d05, 0x01010f03, 0x01010f09,+ 0x01010f0f, 0x01030101, 0x01030103, 0x01030105, 0x01030109, 0x01030301, 0x01030303, 0x0103030b,+ 0x01030501, 0x01030507, 0x0103050f, 0x01030703, 0x0103070b, 0x01030909, 0x01030d03, 0x01030d0b,+ 0x01030f05, 0x01050101, 0x01050103, 0x0105010b, 0x0105010f, 0x01050301, 0x01050307, 0x0105030d,+ 0x01050503, 0x0105050b, 0x01050701, 0x01050709, 0x01050905, 0x0105090b, 0x0105090f, 0x01050b03,+ 0x01050b07, 0x01050f01, 0x01050f07, 0x01070107, 0x01070303, 0x0107030b, 0x01070501, 0x01070505,+ 0x01070703, 0x01070707, 0x0107070d, 0x01070909, 0x01070b01, 0x01070b05, 0x01070d0f, 0x01070f03,+ 0x01070f0b, 0x01090101, 0x01090307, 0x0109030f, 0x01090503, 0x01090509, 0x01090705, 0x01090901,+ 0x01090907, 0x01090b03, 0x01090f01, 0x010b0105, 0x010b0109, 0x010b0501, 0x010b0505, 0x010b050d,+ 0x010b0707, 0x010b0903, 0x010b090b, 0x010b090f, 0x010b0d0d, 0x010b0f07, 0x010d010d, 0x010d0303,+ 0x010d0307, 0x010d0703, 0x010d0b05, 0x010d0f03, 0x010f0101, 0x010f0105, 0x010f0109, 0x010f0501,+ 0x010f0505, 0x010f050d, 0x010f0707, 0x010f0b01, 0x010f0b09, 0x03010101, 0x03010103, 0x03010105,+ 0x03010109, 0x03010301, 0x03010303, 0x03010307, 0x0301030b, 0x0301030f, 0x03010501, 0x03010505,+ 0x03010703, 0x03010709, 0x0301070d, 0x03010b09, 0x03010b0d, 0x03010d03, 0x03010f05, 0x03030101,+ 0x03030103, 0x03030107, 0x0303010d, 0x03030301, 0x03030309, 0x03030503, 0x03030701, 0x03030707,+ 0x03030903, 0x03030b01, 0x03030b05, 0x03030f01, 0x03030f0d, 0x03050101, 0x03050305, 0x0305030b,+ 0x0305030f, 0x03050501, 0x03050509, 0x03050705, 0x03050901, 0x03050907, 0x03050b0b, 0x03050d01,+ 0x03050f05, 0x03070103, 0x03070109, 0x0307010f, 0x03070301, 0x03070307, 0x03070503, 0x0307050f,+ 0x03070701, 0x03070709, 0x03070903, 0x03070d05, 0x03070f01, 0x03090107, 0x0309010b, 0x03090305,+ 0x03090309, 0x03090703, 0x03090707, 0x03090905, 0x0309090d, 0x03090b01, 0x03090b09, 0x030b0103,+ 0x030b0301, 0x030b0307, 0x030b0503, 0x030b0701, 0x030b0705, 0x030b0b03, 0x030d0501, 0x030d0509,+ 0x030d050f, 0x030d0909, 0x030d090d, 0x030f0103, 0x030f0107, 0x030f0301, 0x030f0305, 0x030f0503,+ 0x030f070b, 0x030f0903, 0x030f0d05, 0x030f0f01, 0x05010101, 0x05010103, 0x05010107, 0x0501010b,+ 0x0501010f, 0x05010301, 0x05010305, 0x05010309, 0x0501030d, 0x05010503, 0x05010507, 0x0501050f,+ 0x05010701, 0x05010705, 0x05010903, 0x05010907, 0x0501090b, 0x05010b01, 0x05010b05, 0x05010d0f,+ 0x05010f01, 0x05010f07, 0x05010f0b, 0x05030101, 0x05030105, 0x05030301, 0x05030307, 0x0503030f,+ 0x05030505, 0x0503050b, 0x05030703, 0x05030709, 0x05030905, 0x05030b03, 0x05050103, 0x05050109,+ 0x0505010f, 0x05050503, 0x05050507, 0x05050701, 0x0505070f, 0x05050903, 0x05050b07, 0x05050b0f,+ 0x05050f03, 0x05050f09, 0x05070101, 0x05070105, 0x0507010b, 0x05070303, 0x05070505, 0x05070509,+ 0x05070703, 0x05070707, 0x05070905, 0x05070b01, 0x05070d0d, 0x05090103, 0x0509010f, 0x05090501,+ 0x05090507, 0x05090705, 0x0509070b, 0x05090903, 0x05090f05, 0x05090f0b, 0x050b0109, 0x050b0303,+ 0x050b0505, 0x050b070f, 0x050b0901, 0x050b0b07, 0x050b0f01, 0x050d0101, 0x050d0105, 0x050d010f,+ 0x050d0503, 0x050d0b0b, 0x050d0d03, 0x050f010b, 0x050f0303, 0x050f050d, 0x050f0701, 0x050f0907,+ 0x050f0b01, 0x07010105, 0x07010303, 0x07010307, 0x0701030b, 0x0701030f, 0x07010505, 0x07010703,+ 0x07010707, 0x0701070b, 0x07010905, 0x07010909, 0x0701090f, 0x07010b03, 0x07010d07, 0x07010f03,+ 0x07030103, 0x07030107, 0x0703010b, 0x07030309, 0x07030503, 0x07030507, 0x07030901, 0x07030d01,+ 0x07030f05, 0x07030f0d, 0x07050101, 0x07050305, 0x07050501, 0x07050705, 0x07050709, 0x07050b01,+ 0x07070103, 0x07070301, 0x07070309, 0x07070503, 0x07070507, 0x0707050f, 0x07070701, 0x07070903,+ 0x07070907, 0x0707090f, 0x07070b0b, 0x07070f07, 0x07090107, 0x07090303, 0x0709030d, 0x07090505,+ 0x07090703, 0x07090b05, 0x07090d01, 0x07090d09, 0x070b0103, 0x070b0301, 0x070b0305, 0x070b050b,+ 0x070b0705, 0x070b0909, 0x070b0b0d, 0x070b0f07, 0x070d030d, 0x070d0903, 0x070f0103, 0x070f0107,+ 0x070f0501, 0x070f0505, 0x070f070b, 0x09010101, 0x09010109, 0x09010305, 0x09010501, 0x09010509,+ 0x0901050f, 0x09010705, 0x09010903, 0x09010b01, 0x09010f01, 0x09030105, 0x0903010f, 0x09030303,+ 0x09030307, 0x09030505, 0x09030701, 0x0903070b, 0x09030907, 0x09030b03, 0x09030b0b, 0x09050103,+ 0x09050107, 0x09050301, 0x0905030b, 0x09050503, 0x09050707, 0x09050901, 0x09050b0f, 0x09050d05,+ 0x09050f01, 0x09070109, 0x09070303, 0x09070307, 0x09070501, 0x09070505, 0x09070703, 0x0907070b,+ 0x09090101, 0x09090105, 0x09090509, 0x0909070f, 0x09090901, 0x09090f03, 0x090b010b, 0x090b010f,+ 0x090b0503, 0x090b0d05, 0x090d0307, 0x090d0709, 0x090d0d01, 0x090f0301, 0x090f030b, 0x090f0701,+ 0x090f0907, 0x090f0b03, 0x0b010105, 0x0b010301, 0x0b010309, 0x0b010505, 0x0b010901, 0x0b010909,+ 0x0b01090f, 0x0b010b05, 0x0b010d0d, 0x0b010f09, 0x0b030103, 0x0b030107, 0x0b03010b, 0x0b030305,+ 0x0b030503, 0x0b030705, 0x0b030f05, 0x0b050101, 0x0b050303, 0x0b050507, 0x0b050701, 0x0b05070d,+ 0x0b050b07, 0x0b070105, 0x0b07010f, 0x0b070301, 0x0b07050f, 0x0b070909, 0x0b070b03, 0x0b070d0b,+ 0x0b070f07, 0x0b090103, 0x0b090109, 0x0b090501, 0x0b090705, 0x0b09090d, 0x0b0b0305, 0x0b0b050d,+ 0x0b0b0b03, 0x0b0b0b07, 0x0b0d0905, 0x0b0f0105, 0x0b0f0109, 0x0b0f0505, 0x0d010303, 0x0d010307,+ 0x0d01030b, 0x0d010703, 0x0d010707, 0x0d010d01, 0x0d030101, 0x0d030501, 0x0d03050f, 0x0d030d09,+ 0x0d050305, 0x0d050709, 0x0d050905, 0x0d050b0b, 0x0d050d05, 0x0d050f01, 0x0d070101, 0x0d070309,+ 0x0d070503, 0x0d070901, 0x0d09050b, 0x0d090907, 0x0d090d05, 0x0d0b0101, 0x0d0b0107, 0x0d0b0709,+ 0x0d0b0d01, 0x0d0d010b, 0x0d0d0901, 0x0d0f0303, 0x0d0f0307, 0x0f010101, 0x0f010109, 0x0f01010f,+ 0x0f010501, 0x0f010505, 0x0f01070d, 0x0f010901, 0x0f010b09, 0x0f010d05, 0x0f030105, 0x0f030303,+ 0x0f030509, 0x0f030907, 0x0f03090b, 0x0f050103, 0x0f050109, 0x0f050301, 0x0f05030d, 0x0f050503,+ 0x0f050701, 0x0f050b03, 0x0f070105, 0x0f070705, 0x0f07070b, 0x0f070b07, 0x0f090103, 0x0f09010b,+ 0x0f090307, 0x0f090501, 0x0f090b01, 0x0f0b0505, 0x0f0b0905, 0x0f0d0105, 0x0f0d0703, 0x0f0f0101,+GGML_TABLE_END()++// TODO: fix name to kvalues_iq4_nl+GGML_TABLE_BEGIN(int8_t, kvalues_iq4nl, 16)+ -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,+GGML_TABLE_END()++// e2m1 values (doubled), shared by MXFP4 and NVFP4+// ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf+GGML_TABLE_BEGIN(int8_t, kvalues_fp4, 16)+ 0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12,+GGML_TABLE_END()+#define kvalues_mxfp4 kvalues_fp4++#define NGRID_IQ1S 2048+#define IQ1S_DELTA 0.125f+#define IQ1M_DELTA 0.125f+#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP)+GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)+ 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,+ 0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,+ 0xffffffffff01ff01, 0xffffffffff0101ff, 0xffffffffff010101, 0xffffffff00ff0000,+ 0xffffffff0000ff00, 0xffffffff000000ff, 0xffffffff00000001, 0xffffffff00010000,+ 0xffffffff01ffffff, 0xffffffff01ffff01, 0xffffffff01ff01ff, 0xffffffff01ff0101,+ 0xffffffff01000000, 0xffffffff0101ffff, 0xffffffff0101ff01, 0xffffffff010101ff,+ 0xffffffff01010101, 0xffffff00ffff00ff, 0xffffff00ffff0000, 0xffffff00ff00ff00,+ 0xffffff00ff0000ff, 0xffffff00ff000001, 0xffffff00ff000100, 0xffffff00ff000101,+ 0xffffff00ff010000, 0xffffff0000ffff00, 0xffffff0000ff0001, 0xffffff0000ff0100,+ 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff000001ff00,+ 0xffffff00000100ff, 0xffffff0000010001, 0xffffff00000101ff, 0xffffff0001ff0000,+ 0xffffff000100ff00, 0xffffff00010000ff, 0xffffff0001000001, 0xffffff0001010000,+ 0xffffff01ffffffff, 0xffffff01ffffff01, 0xffffff01ffff01ff, 0xffffff01ffff0101,+ 0xffffff01ff000000, 0xffffff01ff01ffff, 0xffffff01ff01ff01, 0xffffff01ff0101ff,+ 0xffffff01ff010101, 0xffffff0100ff0000, 0xffffff010000ff00, 0xffffff0100000100,+ 0xffffff01000100ff, 0xffffff0100010100, 0xffffff0101ffffff, 0xffffff0101ffff01,+ 0xffffff0101ff01ff, 0xffffff0101ff0101, 0xffffff010100ff00, 0xffffff0101000000,+ 0xffffff0101000100, 0xffffff010101ffff, 0xffffff010101ff01, 0xffffff01010101ff,+ 0xffffff0101010101, 0xffff00ffff00ff00, 0xffff00ffff0000ff, 0xffff00ffff000001,+ 0xffff00ffff010000, 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000,+ 0xffff00ff00000101, 0xffff00ff000100ff, 0xffff00ff00010000, 0xffff00ff0100ff00,+ 0xffff00ff01000100, 0xffff00ff01010000, 0xffff0000ffffff00, 0xffff0000ffff00ff,+ 0xffff0000ffff0000, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff0000ff0001ff,+ 0xffff0000ff000101, 0xffff0000ff010100, 0xffff000000ffffff, 0xffff000000ff0000,+ 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff,+ 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff,+ 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff000000010101,+ 0xffff000001ffff00, 0xffff00000100ff00, 0xffff000001000000, 0xffff0000010001ff,+ 0xffff000001000101, 0xffff00000101ff00, 0xffff0000010100ff, 0xffff000001010000,+ 0xffff000001010001, 0xffff000001010100, 0xffff0001ff0000ff, 0xffff0001ff000100,+ 0xffff000100ffff00, 0xffff000100ff00ff, 0xffff00010000ffff, 0xffff00010000ff01,+ 0xffff000100000000, 0xffff0001000001ff, 0xffff00010001ffff, 0xffff00010001ff00,+ 0xffff000100010001, 0xffff000100010100, 0xffff000101ff0000, 0xffff00010100ff00,+ 0xffff0001010000ff, 0xffff000101000100, 0xffff01ffffffffff, 0xffff01ffffffff01,+ 0xffff01ffffff01ff, 0xffff01ffffff0101, 0xffff01ffff000000, 0xffff01ffff01ffff,+ 0xffff01ffff01ff01, 0xffff01ffff0101ff, 0xffff01ffff010101, 0xffff01ff00ff0000,+ 0xffff01ff0000ff00, 0xffff01ff00000001, 0xffff01ff00010000, 0xffff01ff01ffffff,+ 0xffff01ff01ffff01, 0xffff01ff01ff01ff, 0xffff01ff01ff0101, 0xffff01ff01000000,+ 0xffff01ff0101ffff, 0xffff01ff0101ff01, 0xffff01ff010101ff, 0xffff01ff01010101,+ 0xffff0100ffff0000, 0xffff0100ff00ff00, 0xffff0100ff0000ff, 0xffff0100ff000100,+ 0xffff0100ff0100ff, 0xffff0100ff010000, 0xffff010000ffff00, 0xffff01000000ffff,+ 0xffff01000000ff00, 0xffff010000000000, 0xffff01000001ff00, 0xffff0100000100ff,+ 0xffff010000010100, 0xffff01000100ff00, 0xffff0100010000ff, 0xffff010001000001,+ 0xffff010001000100, 0xffff010001010000, 0xffff0101ffffffff, 0xffff0101ffffff01,+ 0xffff0101ffff01ff, 0xffff0101ffff0101, 0xffff0101ff000000, 0xffff0101ff01ffff,+ 0xffff0101ff01ff01, 0xffff0101ff0101ff, 0xffff0101ff010101, 0xffff010100ff0000,+ 0xffff01010000ff00, 0xffff010100000100, 0xffff01010001ff00, 0xffff010100010000,+ 0xffff010101ffffff, 0xffff010101ffff01, 0xffff010101ff0000, 0xffff010101ff01ff,+ 0xffff010101ff0101, 0xffff010101000000, 0xffff01010101ffff, 0xffff01010101ff01,+ 0xffff0101010101ff, 0xffff010101010101, 0xff00ffffff00ffff, 0xff00ffffff00ff00,+ 0xff00ffffff0000ff, 0xff00ffffff000100, 0xff00ffffff0100ff, 0xff00ffffff010000,+ 0xff00ffff00ffff00, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, 0xff00ffff00000000,+ 0xff00ffff000001ff, 0xff00ffff0001ff00, 0xff00ffff000100ff, 0xff00ffff00010000,+ 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff010000ff, 0xff00ffff01000001,+ 0xff00ffff0101ff00, 0xff00ffff01010000, 0xff00ff00ffffff00, 0xff00ff00ffff00ff,+ 0xff00ff00ffff0001, 0xff00ff00ffff0100, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01,+ 0xff00ff00ff000000, 0xff00ff00ff0001ff, 0xff00ff00ff01ff00, 0xff00ff00ff0100ff,+ 0xff00ff00ff010100, 0xff00ff0000ff0000, 0xff00ff0000ff0101, 0xff00ff000000ffff,+ 0xff00ff000000ff00, 0xff00ff000000ff01, 0xff00ff00000000ff, 0xff00ff0000000000,+ 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, 0xff00ff0000010000,+ 0xff00ff0001ff00ff, 0xff00ff000100ff01, 0xff00ff0001000000, 0xff00ff000101ff00,+ 0xff00ff00010100ff, 0xff00ff01ff00ff00, 0xff00ff01ff0000ff, 0xff00ff01ff000001,+ 0xff00ff01ff010000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, 0xff00ff0100ff0100,+ 0xff00ff010000ff01, 0xff00ff0100000000, 0xff00ff01000001ff, 0xff00ff0100000101,+ 0xff00ff01000100ff, 0xff00ff0100010001, 0xff00ff0101ff0000, 0xff00ff010100ff00,+ 0xff00ff01010000ff, 0xff00ff0101000001, 0xff00ff0101010000, 0xff0000ffffffff00,+ 0xff0000ffffff0001, 0xff0000ffffff0100, 0xff0000ffff0000ff, 0xff0000ffff000000,+ 0xff0000ffff0001ff, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ffff010001,+ 0xff0000ff00ffff00, 0xff0000ff00ff0000, 0xff0000ff00ff0001, 0xff0000ff00ff01ff,+ 0xff0000ff00ff0101, 0xff0000ff0000ff00, 0xff0000ff000000ff, 0xff0000ff00000000,+ 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff0001ff01, 0xff0000ff00010000,+ 0xff0000ff000101ff, 0xff0000ff01ff00ff, 0xff0000ff01ff0100, 0xff0000ff0100ffff,+ 0xff0000ff010000ff, 0xff0000ff01000000, 0xff0000ff010001ff, 0xff0000ff01000100,+ 0xff0000ff01000101, 0xff0000ff0101ff00, 0xff0000ff010100ff, 0xff0000ff01010000,+ 0xff0000ff01010100, 0xff000000ffffff01, 0xff000000ffff0000, 0xff000000ffff0101,+ 0xff000000ff00ff00, 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001,+ 0xff000000ff000100, 0xff000000ff01ffff, 0xff000000ff01ff01, 0xff000000ff010000,+ 0xff000000ff0101ff, 0xff000000ff010101, 0xff00000000ffff00, 0xff00000000ff00ff,+ 0xff00000000ff0000, 0xff00000000ff0001, 0xff0000000000ff00, 0xff0000000000ff01,+ 0xff000000000000ff, 0xff00000000000000, 0xff00000000000001, 0xff00000000000100,+ 0xff00000000000101, 0xff0000000001ff00, 0xff000000000100ff, 0xff00000000010000,+ 0xff00000000010001, 0xff00000000010100, 0xff00000001ffffff, 0xff00000001ffff01,+ 0xff00000001ff00ff, 0xff00000001ff0000, 0xff00000001ff01ff, 0xff00000001ff0101,+ 0xff0000000100ffff, 0xff0000000100ff00, 0xff000000010000ff, 0xff00000001000000,+ 0xff00000001000001, 0xff00000001000100, 0xff00000001000101, 0xff0000000101ffff,+ 0xff0000000101ff01, 0xff00000001010000, 0xff000001ffffff00, 0xff000001ffff00ff,+ 0xff000001ffff0000, 0xff000001ffff0001, 0xff000001ff000000, 0xff000001ff000001,+ 0xff000001ff0001ff, 0xff000001ff000101, 0xff000001ff01ff00, 0xff000001ff010001,+ 0xff00000100ffffff, 0xff00000100ffff01, 0xff00000100ff00ff, 0xff00000100ff0000,+ 0xff00000100ff01ff, 0xff00000100ff0101, 0xff0000010000ff00, 0xff00000100000000,+ 0xff00000100000001, 0xff000001000001ff, 0xff00000100000100, 0xff0000010001ff00,+ 0xff000001000100ff, 0xff00000100010000, 0xff000001000101ff, 0xff00000100010100,+ 0xff00000100010101, 0xff00000101ff0001, 0xff00000101ff0101, 0xff0000010100ff01,+ 0xff00000101000000, 0xff000001010100ff, 0xff00000101010100, 0xff0001ffff00ff00,+ 0xff0001ffff000001, 0xff0001ffff010000, 0xff0001ff00ffff00, 0xff0001ff00ff00ff,+ 0xff0001ff00ff0001, 0xff0001ff00ff0100, 0xff0001ff0000ffff, 0xff0001ff00000000,+ 0xff0001ff000001ff, 0xff0001ff00000101, 0xff0001ff0001ffff, 0xff0001ff0001ff00,+ 0xff0001ff000100ff, 0xff0001ff00010001, 0xff0001ff00010100, 0xff0001ff01ff0000,+ 0xff0001ff0100ff00, 0xff0001ff010000ff, 0xff0001ff01010000, 0xff000100ff00ffff,+ 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff000101, 0xff000100ff01ff00,+ 0xff000100ff010000, 0xff00010000ffff01, 0xff00010000ff00ff, 0xff00010000ff0000,+ 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000,+ 0xff00010000000001, 0xff00010000000100, 0xff00010000000101, 0xff0001000001ffff,+ 0xff00010000010000, 0xff00010000010101, 0xff00010001ff0100, 0xff0001000100ff00,+ 0xff0001000100ff01, 0xff00010001000000, 0xff000100010001ff, 0xff0001000101ff00,+ 0xff00010001010001, 0xff00010001010100, 0xff000101ffff0100, 0xff000101ff000001,+ 0xff000101ff0100ff, 0xff000101ff010001, 0xff00010100ff00ff, 0xff00010100ff0001,+ 0xff00010100ff0100, 0xff0001010000ffff, 0xff0001010000ff01, 0xff00010100000000,+ 0xff000101000001ff, 0xff0001010001ff00, 0xff00010100010001, 0xff00010100010100,+ 0xff00010101ff0000, 0xff0001010100ff00, 0xff00010101000001, 0xff00010101000101,+ 0xff01ffffffffffff, 0xff01ffffffffff01, 0xff01ffffffff01ff, 0xff01ffffffff0101,+ 0xff01ffffff000000, 0xff01ffffff01ffff, 0xff01ffffff01ff01, 0xff01ffffff010000,+ 0xff01ffffff0101ff, 0xff01ffffff010101, 0xff01ffff00ff0000, 0xff01ffff0000ff00,+ 0xff01ffff00000100, 0xff01ffff0001ff00, 0xff01ffff00010000, 0xff01ffff01ffffff,+ 0xff01ffff01ffff01, 0xff01ffff01ff01ff, 0xff01ffff01ff0101, 0xff01ffff01000000,+ 0xff01ffff0101ffff, 0xff01ffff0101ff01, 0xff01ffff01010000, 0xff01ffff010101ff,+ 0xff01ffff01010101, 0xff01ff00ffff0000, 0xff01ff00ff00ff00, 0xff01ff00ff0000ff,+ 0xff01ff00ff000100, 0xff01ff00ff010000, 0xff01ff0000ffff01, 0xff01ff0000ff00ff,+ 0xff01ff0000ff0100, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101,+ 0xff01ff000001ff00, 0xff01ff00000100ff, 0xff01ff0000010000, 0xff01ff0000010001,+ 0xff01ff0001ff0000, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001000100,+ 0xff01ff0001010000, 0xff01ff01ffffff00, 0xff01ff01ffff01ff, 0xff01ff01ffff0101,+ 0xff01ff01ff00ff00, 0xff01ff01ff000000, 0xff01ff01ff01ffff, 0xff01ff01ff01ff01,+ 0xff01ff01ff0101ff, 0xff01ff01ff010101, 0xff01ff0100ff0000, 0xff01ff010000ff00,+ 0xff01ff0100000001, 0xff01ff0100000100, 0xff01ff0100010000, 0xff01ff0101ffff00,+ 0xff01ff0101ff01ff, 0xff01ff0101ff0101, 0xff01ff010100ff00, 0xff01ff0101000000,+ 0xff01ff010101ffff, 0xff01ff010101ff01, 0xff01ff01010101ff, 0xff01ff0101010101,+ 0xff0100ffffff0000, 0xff0100ffff0000ff, 0xff0100ffff000001, 0xff0100ffff000100,+ 0xff0100ffff010000, 0xff0100ff00ff00ff, 0xff0100ff00ff0000, 0xff0100ff00ff0001,+ 0xff0100ff00ff0100, 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff,+ 0xff0100ff00000101, 0xff0100ff00010001, 0xff0100ff01ff0000, 0xff0100ff0100ff00,+ 0xff0100ff010000ff, 0xff0100ff01000100, 0xff0100ff0101ff00, 0xff0100ff01010000,+ 0xff010000ffff0100, 0xff010000ff000000, 0xff010000ff01ff00, 0xff010000ff010100,+ 0xff01000000ffffff, 0xff01000000ff0000, 0xff01000000ff01ff, 0xff0100000000ff00,+ 0xff010000000000ff, 0xff01000000000000, 0xff01000000000100, 0xff0100000001ff01,+ 0xff01000000010000, 0xff010000000101ff, 0xff01000001ff0100, 0xff0100000100ffff,+ 0xff010000010000ff, 0xff01000001000000, 0xff010000010001ff, 0xff01000001000101,+ 0xff0100000101ff00, 0xff010000010100ff, 0xff01000001010001, 0xff01000001010100,+ 0xff010001ffff0000, 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100,+ 0xff010001ff010000, 0xff01000100ffff00, 0xff01000100ff0100, 0xff01000100000000,+ 0xff0100010001ffff, 0xff0100010001ff00, 0xff01000100010100, 0xff01000101ff00ff,+ 0xff01000101ff0001, 0xff0100010100ffff, 0xff01000101000101, 0xff0101ffffffffff,+ 0xff0101ffffffff01, 0xff0101ffffff01ff, 0xff0101ffffff0101, 0xff0101ffff000000,+ 0xff0101ffff01ffff, 0xff0101ffff01ff01, 0xff0101ffff0101ff, 0xff0101ffff010101,+ 0xff0101ff00ff0000, 0xff0101ff0000ff00, 0xff0101ff000000ff, 0xff0101ff00010000,+ 0xff0101ff01ffffff, 0xff0101ff01ffff01, 0xff0101ff01ff01ff, 0xff0101ff01ff0101,+ 0xff0101ff0101ffff, 0xff0101ff0101ff01, 0xff0101ff010101ff, 0xff0101ff01010101,+ 0xff010100ffff0100, 0xff010100ff00ff00, 0xff010100ff0000ff, 0xff010100ff000100,+ 0xff010100ff010000, 0xff01010000ff0001, 0xff01010000ff0100, 0xff0101000000ff01,+ 0xff01010000000000, 0xff0101000001ff00, 0xff010100000100ff, 0xff01010000010001,+ 0xff01010000010100, 0xff01010001ff0000, 0xff0101000100ffff, 0xff01010001000001,+ 0xff01010001000100, 0xff010100010100ff, 0xff01010001010000, 0xff010101ffffffff,+ 0xff010101ffffff01, 0xff010101ffff01ff, 0xff010101ffff0101, 0xff010101ff01ffff,+ 0xff010101ff01ff01, 0xff010101ff0101ff, 0xff010101ff010101, 0xff01010100ff0000,+ 0xff0101010000ff00, 0xff01010100000001, 0xff01010100000100, 0xff01010100010000,+ 0xff01010101ffffff, 0xff01010101ffff01, 0xff01010101ff01ff, 0xff01010101ff0101,+ 0xff01010101000000, 0xff0101010101ffff, 0xff0101010101ff01, 0xff010101010101ff,+ 0xff01010101010101, 0x00ffffffffff0000, 0x00ffffffff00ff00, 0x00ffffffff000001,+ 0x00ffffffff010000, 0x00ffffff00ff0100, 0x00ffffff0000ff01, 0x00ffffff00000000,+ 0x00ffffff000001ff, 0x00ffffff00000101, 0x00ffffff0001ff00, 0x00ffffff000100ff,+ 0x00ffffff00010001, 0x00ffffff010000ff, 0x00ffffff01000100, 0x00ffffff0101ff00,+ 0x00ffffff01010001, 0x00ffff00ffffffff, 0x00ffff00ffffff00, 0x00ffff00ffff00ff,+ 0x00ffff00ffff0001, 0x00ffff00ffff0100, 0x00ffff00ff00ff01, 0x00ffff00ff000000,+ 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff000101, 0x00ffff00ff01ff00,+ 0x00ffff00ff010001, 0x00ffff00ff010100, 0x00ffff0000ff0000, 0x00ffff0000ff01ff,+ 0x00ffff0000ff0101, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000,+ 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000,+ 0x00ffff00000101ff, 0x00ffff0000010101, 0x00ffff0001ffff00, 0x00ffff0001ff00ff,+ 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff000100ff01, 0x00ffff0001000000,+ 0x00ffff000101ffff, 0x00ffff000101ff00, 0x00ffff000101ff01, 0x00ffff01ffff0000,+ 0x00ffff01ff00ff00, 0x00ffff01ff0000ff, 0x00ffff01ff000001, 0x00ffff01ff010000,+ 0x00ffff0100ffff00, 0x00ffff010000ff01, 0x00ffff0100000000, 0x00ffff0100000101,+ 0x00ffff01000100ff, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ffff01010000ff,+ 0x00ffff0101010000, 0x00ff00ffffffff00, 0x00ff00ffff000000, 0x00ff00ffff000100,+ 0x00ff00ffff010100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, 0x00ff00ff00ff0101,+ 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001,+ 0x00ff00ff0001ff00, 0x00ff00ff0001ff01, 0x00ff00ff00010000, 0x00ff00ff000101ff,+ 0x00ff00ff00010101, 0x00ff00ff01ffff00, 0x00ff00ff01ff0001, 0x00ff00ff01ff0100,+ 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, 0x00ff00ff0101ffff,+ 0x00ff00ff0101ff00, 0x00ff00ff01010100, 0x00ff0000ffffff00, 0x00ff0000ffffff01,+ 0x00ff0000ffff0000, 0x00ff0000ffff0101, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff,+ 0x00ff0000ff000000, 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff,+ 0x00ff0000ff010000, 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff,+ 0x00ff000000ff0000, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff,+ 0x00ff00000000ff00, 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001,+ 0x00ff0000000001ff, 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff,+ 0x00ff000000010000, 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01,+ 0x00ff000001ff00ff, 0x00ff000001ff0000, 0x00ff000001ff01ff, 0x00ff00000100ff00,+ 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, 0x00ff000001000100,+ 0x00ff000001000101, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101,+ 0x00ff0001ffffff00, 0x00ff0001ffff0000, 0x00ff0001ffff0100, 0x00ff0001ff0000ff,+ 0x00ff0001ff000000, 0x00ff0001ff0001ff, 0x00ff0001ff000101, 0x00ff0001ff01ff00,+ 0x00ff0001ff0100ff, 0x00ff0001ff010100, 0x00ff000100ffffff, 0x00ff000100ffff01,+ 0x00ff000100ff0000, 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00,+ 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100,+ 0x00ff00010001ff01, 0x00ff000100010000, 0x00ff0001000101ff, 0x00ff000101ffff00,+ 0x00ff000101ff0000, 0x00ff000101ff0101, 0x00ff0001010000ff, 0x00ff000101000000,+ 0x00ff00010101ff00, 0x00ff0001010100ff, 0x00ff000101010001, 0x00ff01ffffff0000,+ 0x00ff01ffff00ff00, 0x00ff01ffff000000, 0x00ff01ffff000101, 0x00ff01ffff010000,+ 0x00ff01ff00ffff01, 0x00ff01ff00ff0100, 0x00ff01ff0000ffff, 0x00ff01ff00000000,+ 0x00ff01ff000001ff, 0x00ff01ff0001ff00, 0x00ff01ff000100ff, 0x00ff01ff00010001,+ 0x00ff01ff00010100, 0x00ff01ff01ff0000, 0x00ff01ff0100ff00, 0x00ff01ff010000ff,+ 0x00ff01ff01000001, 0x00ff01ff01000100, 0x00ff01ff01010000, 0x00ff0100ffffff00,+ 0x00ff0100ffff0000, 0x00ff0100ffff0001, 0x00ff0100ffff0101, 0x00ff0100ff00ffff,+ 0x00ff0100ff0000ff, 0x00ff0100ff000000, 0x00ff0100ff0001ff, 0x00ff0100ff01ff00,+ 0x00ff0100ff0100ff, 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000,+ 0x00ff010000ff0101, 0x00ff01000000ff00, 0x00ff01000000ff01, 0x00ff0100000000ff,+ 0x00ff010000000000, 0x00ff010000000001, 0x00ff010000000100, 0x00ff01000001ffff,+ 0x00ff01000001ff01, 0x00ff010000010000, 0x00ff010000010001, 0x00ff010000010101,+ 0x00ff010001ff0001, 0x00ff010001ff0100, 0x00ff01000100ff01, 0x00ff010001000000,+ 0x00ff010001000001, 0x00ff0100010001ff, 0x00ff01000101ff00, 0x00ff0100010100ff,+ 0x00ff010001010001, 0x00ff010001010100, 0x00ff0101ff000001, 0x00ff010100ff00ff,+ 0x00ff010100ff0001, 0x00ff010100ff0100, 0x00ff010100000000, 0x00ff0101000001ff,+ 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x00ff0101010000ff,+ 0x00ff010101010000, 0x0000ffffffffff00, 0x0000ffffffff00ff, 0x0000ffffffff0000,+ 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000,+ 0x0000ffffff000101, 0x0000ffffff01ff00, 0x0000ffffff0100ff, 0x0000ffffff010100,+ 0x0000ffff00ffffff, 0x0000ffff00ff0000, 0x0000ffff00ff01ff, 0x0000ffff0000ff00,+ 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100,+ 0x0000ffff00010000, 0x0000ffff000101ff, 0x0000ffff01ff0001, 0x0000ffff01ff0100,+ 0x0000ffff01000000, 0x0000ffff010001ff, 0x0000ffff0101ffff, 0x0000ffff0101ff00,+ 0x0000ffff01010001, 0x0000ffff01010100, 0x0000ff00ffff0000, 0x0000ff00ffff01ff,+ 0x0000ff00ffff0100, 0x0000ff00ffff0101, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff,+ 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff0001ff, 0x0000ff00ff000100,+ 0x0000ff00ff01ffff, 0x0000ff00ff010000, 0x0000ff00ff010001, 0x0000ff00ff0101ff,+ 0x0000ff00ff010101, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, 0x0000ff0000ff0000,+ 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, 0x0000ff000000ff00,+ 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, 0x0000ff0000000001,+ 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, 0x0000ff000001ff00,+ 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, 0x0000ff0000010100,+ 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff,+ 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff000101ffff,+ 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ffffff00, 0x0000ff01ffff0001,+ 0x0000ff01ff00ff01, 0x0000ff01ff000000, 0x0000ff01ff000101, 0x0000ff01ff01ff00,+ 0x0000ff01ff0100ff, 0x0000ff0100ffff01, 0x0000ff0100ff0000, 0x0000ff0100ff0101,+ 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001,+ 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, 0x0000ff0101ff0000,+ 0x0000ff010100ffff, 0x0000ff010100ff01, 0x0000ff0101000000, 0x0000ff0101000100,+ 0x0000ff0101000101, 0x0000ff01010100ff, 0x000000ffffff00ff, 0x000000ffffff0000,+ 0x000000ffff00ff00, 0x000000ffff0000ff, 0x000000ffff000000, 0x000000ffff000001,+ 0x000000ffff0001ff, 0x000000ffff000100, 0x000000ffff01ff00, 0x000000ffff010000,+ 0x000000ffff0101ff, 0x000000ffff010101, 0x000000ff00ffff00, 0x000000ff00ff00ff,+ 0x000000ff00ff0000, 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101,+ 0x000000ff0000ffff, 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000,+ 0x000000ff00000001, 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101,+ 0x000000ff0001ff00, 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000,+ 0x000000ff00010001, 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff01ff01ff,+ 0x000000ff01ff0101, 0x000000ff0100ff00, 0x000000ff010000ff, 0x000000ff01000000,+ 0x000000ff01000001, 0x000000ff01000100, 0x000000ff0101ff00, 0x000000ff010100ff,+ 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01,+ 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100,+ 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff,+ 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101,+ 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001,+ 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01,+ 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff,+ 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01,+ 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff,+ 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00,+ 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff,+ 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff,+ 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff,+ 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001,+ 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff,+ 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffffff,+ 0x00000001ffffff00, 0x00000001ffffff01, 0x00000001ffff00ff, 0x00000001ffff0001,+ 0x00000001ffff01ff, 0x00000001ffff0100, 0x00000001ff00ff00, 0x00000001ff0000ff,+ 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, 0x00000001ff01ffff,+ 0x00000001ff01ff00, 0x00000001ff01ff01, 0x00000001ff0100ff, 0x00000001ff010000,+ 0x00000001ff010001, 0x00000001ff0101ff, 0x00000001ff010100, 0x0000000100ffff00,+ 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff01ff, 0x0000000100ff0100,+ 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01,+ 0x00000001000000ff, 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff,+ 0x0000000100000100, 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff,+ 0x0000000100010000, 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000,+ 0x0000000101ff0001, 0x0000000101ff01ff, 0x0000000101ff0100, 0x0000000101ff0101,+ 0x000000010100ff00, 0x0000000101000000, 0x0000000101000101, 0x000000010101ff01,+ 0x0000000101010000, 0x0000000101010001, 0x00000001010101ff, 0x0000000101010100,+ 0x000001ffffff00ff, 0x000001ffffff0000, 0x000001ffffff0001, 0x000001ffffff0100,+ 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff0001ff, 0x000001ffff01ff00,+ 0x000001ffff010101, 0x000001ff00ff0000, 0x000001ff00ff01ff, 0x000001ff00ff0101,+ 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, 0x000001ff00000001,+ 0x000001ff000001ff, 0x000001ff00000100, 0x000001ff0001ffff, 0x000001ff0001ff01,+ 0x000001ff000100ff, 0x000001ff00010000, 0x000001ff01ffff01, 0x000001ff01ff0100,+ 0x000001ff0100ffff, 0x000001ff0100ff01, 0x000001ff01000000, 0x000001ff010001ff,+ 0x000001ff0101ff00, 0x000001ff01010100, 0x00000100ffffff00, 0x00000100ffffff01,+ 0x00000100ffff0000, 0x00000100ffff0101, 0x00000100ff00ff00, 0x00000100ff0000ff,+ 0x00000100ff000000, 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000,+ 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001,+ 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01,+ 0x00000100000000ff, 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff,+ 0x0000010000000100, 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff,+ 0x0000010000010000, 0x0000010000010001, 0x0000010000010100, 0x0000010001ffff00,+ 0x0000010001ff0000, 0x0000010001ff0100, 0x000001000100ff00, 0x00000100010000ff,+ 0x0000010001000000, 0x0000010001000001, 0x00000100010001ff, 0x0000010001000100,+ 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ffff01ff, 0x00000101ff000000,+ 0x00000101ff000101, 0x00000101ff01ffff, 0x00000101ff010000, 0x00000101ff010001,+ 0x00000101ff010100, 0x0000010100ff0000, 0x0000010100ff01ff, 0x0000010100ff0100,+ 0x000001010000ff00, 0x0000010100000000, 0x0000010100000001, 0x00000101000001ff,+ 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, 0x00000101000101ff,+ 0x0000010100010101, 0x0000010101ffff00, 0x0000010101ff0101, 0x000001010100ff01,+ 0x0000010101000000, 0x0000010101000001, 0x00000101010001ff, 0x0000010101000101,+ 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, 0x0001ffffff000001,+ 0x0001ffffff000100, 0x0001ffffff010000, 0x0001ffff00ff00ff, 0x0001ffff0000ffff,+ 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff00000101,+ 0x0001ffff0001ff00, 0x0001ffff000100ff, 0x0001ffff00010001, 0x0001ffff00010100,+ 0x0001ffff01ffff00, 0x0001ffff01000001, 0x0001ffff01010000, 0x0001ff00ffffff00,+ 0x0001ff00ffff00ff, 0x0001ff00ffff0001, 0x0001ff00ffff0100, 0x0001ff00ff00ff01,+ 0x0001ff00ff000000, 0x0001ff00ff01ff00, 0x0001ff00ff01ff01, 0x0001ff00ff010001,+ 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff0000ff0100, 0x0001ff000000ff00,+ 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000,+ 0x0001ff0000010001, 0x0001ff0000010101, 0x0001ff0001ff00ff, 0x0001ff0001ff0101,+ 0x0001ff000100ff01, 0x0001ff0001000000, 0x0001ff000101ff00, 0x0001ff0001010001,+ 0x0001ff0001010100, 0x0001ff01ff00ff00, 0x0001ff01ff000001, 0x0001ff01ff000100,+ 0x0001ff0100ffffff, 0x0001ff0100ffff00, 0x0001ff0100ff0001, 0x0001ff0100000000,+ 0x0001ff0100000001, 0x0001ff01000001ff, 0x0001ff010001ffff, 0x0001ff0101ff0000,+ 0x0001ff010100ff00, 0x0001ff0101000001, 0x0001ff0101010000, 0x000100ffff00ff00,+ 0x000100ffff00ff01, 0x000100ffff000000, 0x000100ffff000001, 0x000100ffff000101,+ 0x000100ffff01ff00, 0x000100ffff010001, 0x000100ffff010100, 0x000100ff00ffffff,+ 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff00ff0101,+ 0x000100ff0000ff00, 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000001,+ 0x000100ff00000100, 0x000100ff00000101, 0x000100ff0001ffff, 0x000100ff0001ff01,+ 0x000100ff00010000, 0x000100ff01ff00ff, 0x000100ff01ff0000, 0x000100ff01ff0100,+ 0x000100ff0100ffff, 0x000100ff0100ff01, 0x000100ff010000ff, 0x000100ff01000000,+ 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff01000101, 0x000100ff0101ff00,+ 0x000100ff010100ff, 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff01ff,+ 0x00010000ffff0101, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001,+ 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0000, 0x0001000000ff0001,+ 0x0001000000ff0100, 0x000100000000ffff, 0x000100000000ff00, 0x00010000000000ff,+ 0x0001000000000000, 0x0001000000000001, 0x0001000000000100, 0x000100000001ff00,+ 0x00010000000100ff, 0x0001000000010000, 0x0001000000010001, 0x0001000000010100,+ 0x0001000001ff0001, 0x0001000001ff0100, 0x0001000001ff0101, 0x000100000100ff00,+ 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101,+ 0x000100000101ff01, 0x0001000001010000, 0x0001000001010001, 0x00010000010101ff,+ 0x00010001ffffff01, 0x00010001ffff0100, 0x00010001ff000000, 0x00010001ff01ffff,+ 0x00010001ff010001, 0x00010001ff0101ff, 0x00010001ff010100, 0x0001000100ffffff,+ 0x0001000100ff0000, 0x0001000100ff01ff, 0x0001000100ff0101, 0x000100010000ff00,+ 0x00010001000000ff, 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff,+ 0x0001000100000101, 0x000100010001ffff, 0x0001000100010000, 0x00010001000101ff,+ 0x0001000101ffffff, 0x0001000101ffff01, 0x0001000101ff0000, 0x0001000101ff0101,+ 0x00010001010000ff, 0x0001000101000001, 0x00010001010001ff, 0x0001000101000100,+ 0x000100010101ffff, 0x00010001010100ff, 0x0001000101010001, 0x0001000101010101,+ 0x000101ffff000001, 0x000101ffff000100, 0x000101ffff010000, 0x000101ff00ffff00,+ 0x000101ff0000ff01, 0x000101ff00000000, 0x000101ff00000101, 0x000101ff0001ff00,+ 0x000101ff00010100, 0x000101ff01ff0000, 0x000101ff0100ff00, 0x000101ff010001ff,+ 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ffff00ff, 0x00010100ff00ffff,+ 0x00010100ff000000, 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001,+ 0x00010100ff010100, 0x0001010000ffffff, 0x0001010000ffff00, 0x0001010000ff0000,+ 0x0001010000ff0001, 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff,+ 0x0001010000000000, 0x0001010000000001, 0x0001010000000100, 0x000101000001ffff,+ 0x0001010000010000, 0x0001010000010101, 0x0001010001ffff01, 0x0001010001ff00ff,+ 0x0001010001ff0101, 0x0001010001000000, 0x000101000101ff00, 0x00010100010100ff,+ 0x0001010001010000, 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff000001,+ 0x00010101ff0001ff, 0x0001010100ffff00, 0x0001010100ff00ff, 0x0001010100ff0100,+ 0x000101010000ffff, 0x0001010100000000, 0x00010101000001ff, 0x0001010100000101,+ 0x00010101000100ff, 0x0001010100010000, 0x0001010100010100, 0x0001010101ff0001,+ 0x00010101010000ff, 0x00010101010001ff, 0x0001010101000101, 0x0001010101010001,+ 0x01ffffffffffffff, 0x01ffffffffffff01, 0x01ffffffffff01ff, 0x01ffffffffff0101,+ 0x01ffffffff01ffff, 0x01ffffffff01ff01, 0x01ffffffff0101ff, 0x01ffffffff010101,+ 0x01ffffff00ff0000, 0x01ffffff0000ffff, 0x01ffffff0000ff00, 0x01ffffff000000ff,+ 0x01ffffff00000001, 0x01ffffff00000100, 0x01ffffff00010000, 0x01ffffff01ffffff,+ 0x01ffffff01ffff01, 0x01ffffff01ff01ff, 0x01ffffff01ff0101, 0x01ffffff01000000,+ 0x01ffffff0101ffff, 0x01ffffff0101ff01, 0x01ffffff010101ff, 0x01ffffff01010101,+ 0x01ffff00ffff0000, 0x01ffff00ff00ff00, 0x01ffff00ff0000ff, 0x01ffff00ff000001,+ 0x01ffff00ff000100, 0x01ffff00ff010000, 0x01ffff0000ffff00, 0x01ffff0000ff00ff,+ 0x01ffff0000ff0100, 0x01ffff000000ffff, 0x01ffff000000ff01, 0x01ffff0000000000,+ 0x01ffff0000000001, 0x01ffff00000001ff, 0x01ffff0000000100, 0x01ffff00000100ff,+ 0x01ffff0000010001, 0x01ffff0000010100, 0x01ffff0001ff0000, 0x01ffff0001ff0100,+ 0x01ffff00010000ff, 0x01ffff0001000001, 0x01ffff0001000100, 0x01ffff0001010000,+ 0x01ffff01ffffffff, 0x01ffff01ffffff01, 0x01ffff01ffff01ff, 0x01ffff01ffff0101,+ 0x01ffff01ff000000, 0x01ffff01ff01ffff, 0x01ffff01ff01ff01, 0x01ffff01ff0101ff,+ 0x01ffff01ff010101, 0x01ffff010000ff00, 0x01ffff01000000ff, 0x01ffff0100000100,+ 0x01ffff0100010000, 0x01ffff0101ffffff, 0x01ffff0101ffff01, 0x01ffff0101ff01ff,+ 0x01ffff0101ff0101, 0x01ffff0101000000, 0x01ffff010101ffff, 0x01ffff010101ff01,+ 0x01ffff01010101ff, 0x01ffff0101010101, 0x01ff00ffff0000ff, 0x01ff00ffff000100,+ 0x01ff00ff00ffff00, 0x01ff00ff00ff00ff, 0x01ff00ff0000ff00, 0x01ff00ff00000000,+ 0x01ff00ff00000101, 0x01ff00ff0001ff00, 0x01ff00ff000100ff, 0x01ff00ff00010100,+ 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ffffff00, 0x01ff0000ffff0100,+ 0x01ff0000ff00ff01, 0x01ff0000ff000000, 0x01ff0000ff000101, 0x01ff0000ff010001,+ 0x01ff0000ff010100, 0x01ff000000ffffff, 0x01ff000000ffff00, 0x01ff000000ff0000,+ 0x01ff000000ff01ff, 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000,+ 0x01ff000000000001, 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000,+ 0x01ff000000010001, 0x01ff0000000101ff, 0x01ff000000010101, 0x01ff000001ffff00,+ 0x01ff000001ff00ff, 0x01ff000001ff0001, 0x01ff000001ff0100, 0x01ff00000100ffff,+ 0x01ff00000100ff01, 0x01ff000001000000, 0x01ff0000010001ff, 0x01ff000001010001,+ 0x01ff0001ff00ff00, 0x01ff0001ff000001, 0x01ff0001ff000100, 0x01ff0001ff010000,+ 0x01ff000100ffff00, 0x01ff000100ff00ff, 0x01ff000100ff0100, 0x01ff000100ff0101,+ 0x01ff00010000ffff, 0x01ff000100000000, 0x01ff000100000100, 0x01ff000100000101,+ 0x01ff00010001ff00, 0x01ff000100010001, 0x01ff000100010101, 0x01ff000101ff0000,+ 0x01ff00010100ff00, 0x01ff000101000101, 0x01ff0001010100ff, 0x01ff01ffffffffff,+ 0x01ff01ffffffff01, 0x01ff01ffffff01ff, 0x01ff01ffffff0101, 0x01ff01ffff000000,+ 0x01ff01ffff01ffff, 0x01ff01ffff01ff01, 0x01ff01ffff0101ff, 0x01ff01ffff010101,+ 0x01ff01ff00ffff00, 0x01ff01ff00ff0000, 0x01ff01ff0000ff00, 0x01ff01ff000000ff,+ 0x01ff01ff00000100, 0x01ff01ff00010000, 0x01ff01ff00010100, 0x01ff01ff01ffffff,+ 0x01ff01ff01ffff01, 0x01ff01ff01ff01ff, 0x01ff01ff01ff0101, 0x01ff01ff01000000,+ 0x01ff01ff0101ffff, 0x01ff01ff0101ff01, 0x01ff01ff010101ff, 0x01ff01ff01010101,+ 0x01ff0100ffff0000, 0x01ff0100ffff0001, 0x01ff0100ff00ff00, 0x01ff0100ff0000ff,+ 0x01ff0100ff000001, 0x01ff0100ff010000, 0x01ff010000ffff00, 0x01ff010000ff00ff,+ 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff01000000ffff, 0x01ff01000000ff01,+ 0x01ff010000000000, 0x01ff010000000101, 0x01ff01000001ff00, 0x01ff0100000100ff,+ 0x01ff010001ff0000, 0x01ff010001000001, 0x01ff010001000100, 0x01ff010001010000,+ 0x01ff0101ffffffff, 0x01ff0101ffffff01, 0x01ff0101ffff01ff, 0x01ff0101ffff0101,+ 0x01ff0101ff000000, 0x01ff0101ff01ffff, 0x01ff0101ff01ff01, 0x01ff0101ff0101ff,+ 0x01ff0101ff010101, 0x01ff010100ff0000, 0x01ff01010000ff00, 0x01ff0101000000ff,+ 0x01ff010100000001, 0x01ff010101ffffff, 0x01ff010101ffff01, 0x01ff010101ff01ff,+ 0x01ff010101ff0101, 0x01ff010101000000, 0x01ff01010101ffff, 0x01ff01010101ff01,+ 0x01ff0101010101ff, 0x01ff010101010101, 0x0100ffffffff0000, 0x0100ffffff00ff00,+ 0x0100ffffff000001, 0x0100ffffff0001ff, 0x0100ffffff000100, 0x0100ffffff010000,+ 0x0100ffff00ffff00, 0x0100ffff00ff0001, 0x0100ffff00ff0100, 0x0100ffff00000000,+ 0x0100ffff000001ff, 0x0100ffff00000101, 0x0100ffff00010100, 0x0100ffff00010101,+ 0x0100ffff01ff0000, 0x0100ffff0100ff00, 0x0100ffff010000ff, 0x0100ffff01000001,+ 0x0100ffff01000100, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff00ff,+ 0x0100ff00ffff0001, 0x0100ff00ffff0100, 0x0100ff00ff00ffff, 0x0100ff00ff000000,+ 0x0100ff00ff0001ff, 0x0100ff00ff000101, 0x0100ff00ff01ff00, 0x0100ff00ff0100ff,+ 0x0100ff00ff010001, 0x0100ff00ff010100, 0x0100ff0000ffffff, 0x0100ff0000ff0000,+ 0x0100ff000000ffff, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000,+ 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000,+ 0x0100ff0001ff00ff, 0x0100ff0001ff0001, 0x0100ff000100ff01, 0x0100ff0001000000,+ 0x0100ff00010001ff, 0x0100ff000101ff00, 0x0100ff00010100ff, 0x0100ff0001010001,+ 0x0100ff0001010100, 0x0100ff01ffff0000, 0x0100ff01ff00ff00, 0x0100ff01ff0000ff,+ 0x0100ff01ff000100, 0x0100ff01ff010000, 0x0100ff0100ff00ff, 0x0100ff0100ff0001,+ 0x0100ff0100ff0100, 0x0100ff010000ffff, 0x0100ff010000ff01, 0x0100ff0100000000,+ 0x0100ff01000001ff, 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101ff0000,+ 0x0100ff01010000ff, 0x0100ff0101000001, 0x0100ff0101010100, 0x010000ffffffff00,+ 0x010000ffffff00ff, 0x010000ffffff0001, 0x010000ffff00ffff, 0x010000ffff000000,+ 0x010000ffff0001ff, 0x010000ffff010001, 0x010000ff00ffffff, 0x010000ff00ff0101,+ 0x010000ff0000ff00, 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001,+ 0x010000ff000001ff, 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff0001ff00,+ 0x010000ff0001ff01, 0x010000ff00010000, 0x010000ff01ff00ff, 0x010000ff01ff0001,+ 0x010000ff0100ff01, 0x010000ff010000ff, 0x010000ff01000000, 0x010000ff010001ff,+ 0x010000ff0101ff00, 0x010000ff01010100, 0x01000000ffffffff, 0x01000000ffff0000,+ 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ffff, 0x01000000ff00ff00,+ 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100,+ 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010100, 0x01000000ff010101,+ 0x0100000000ffff00, 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001,+ 0x0100000000ff0100, 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01,+ 0x01000000000000ff, 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff,+ 0x0100000000000100, 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff,+ 0x0100000000010000, 0x0100000000010001, 0x0100000000010100, 0x0100000001ffff00,+ 0x0100000001ff0000, 0x0100000001ff01ff, 0x010000000100ff00, 0x010000000100ff01,+ 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, 0x0100000001000100,+ 0x0100000001000101, 0x010000000101ffff, 0x010000000101ff01, 0x0100000001010000,+ 0x01000000010101ff, 0x0100000001010101, 0x01000001ffffff00, 0x01000001ffff00ff,+ 0x01000001ff00ffff, 0x01000001ff000000, 0x01000001ff000100, 0x01000001ff01ffff,+ 0x01000001ff010001, 0x01000001ff010100, 0x0100000100ff0000, 0x0100000100ff01ff,+ 0x0100000100ff0100, 0x010000010000ff00, 0x010000010000ff01, 0x0100000100000000,+ 0x0100000100000001, 0x0100000100000100, 0x0100000100010000, 0x01000001000101ff,+ 0x0100000101ffff01, 0x0100000101ff00ff, 0x0100000101ff0100, 0x0100000101ff0101,+ 0x010000010100ff01, 0x01000001010000ff, 0x0100000101000000, 0x01000001010100ff,+ 0x0100000101010001, 0x0100000101010100, 0x010001ffffff0000, 0x010001ffff000001,+ 0x010001ffff000100, 0x010001ffff010000, 0x010001ff00ffff00, 0x010001ff00ff0001,+ 0x010001ff0000ffff, 0x010001ff0000ff01, 0x010001ff00000000, 0x010001ff00000001,+ 0x010001ff00000101, 0x010001ff000100ff, 0x010001ff00010000, 0x010001ff01ff0000,+ 0x010001ff0100ff00, 0x010001ff01000001, 0x010001ff01000100, 0x010001ff01010000,+ 0x01000100ffff00ff, 0x01000100ffff0001, 0x01000100ffff0100, 0x01000100ff00ffff,+ 0x01000100ff00ff01, 0x01000100ff000000, 0x01000100ff0001ff, 0x01000100ff000101,+ 0x01000100ff01ffff, 0x01000100ff01ff00, 0x01000100ff0100ff, 0x01000100ff010001,+ 0x0100010000ffffff, 0x0100010000ffff01, 0x0100010000ff0000, 0x0100010000ff01ff,+ 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000,+ 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000,+ 0x0100010000010001, 0x0100010000010101, 0x0100010001ffff00, 0x0100010001ff00ff,+ 0x010001000100ffff, 0x010001000100ff01, 0x0100010001000000, 0x0100010001000101,+ 0x010001000101ff00, 0x0100010001010001, 0x01000101ffff0000, 0x01000101ff000000,+ 0x01000101ff010000, 0x0100010100ff00ff, 0x0100010100ff0001, 0x0100010100ff0100,+ 0x010001010000ffff, 0x0100010100000000, 0x01000101000001ff, 0x010001010001ff00,+ 0x0100010101ff0000, 0x010001010100ff00, 0x01000101010000ff, 0x0100010101000000,+ 0x0100010101000001, 0x0101ffffffffffff, 0x0101ffffffffff01, 0x0101ffffffff01ff,+ 0x0101ffffffff0101, 0x0101ffffff000000, 0x0101ffffff01ffff, 0x0101ffffff01ff01,+ 0x0101ffffff0101ff, 0x0101ffffff010101, 0x0101ffff00ff0000, 0x0101ffff0000ff00,+ 0x0101ffff000000ff, 0x0101ffff00000001, 0x0101ffff00000100, 0x0101ffff01ffffff,+ 0x0101ffff01ffff01, 0x0101ffff01ff01ff, 0x0101ffff01ff0101, 0x0101ffff01000000,+ 0x0101ffff0101ffff, 0x0101ffff0101ff01, 0x0101ffff010101ff, 0x0101ffff01010101,+ 0x0101ff00ffff0000, 0x0101ff00ffff0100, 0x0101ff00ff00ff00, 0x0101ff00ff0000ff,+ 0x0101ff00ff000001, 0x0101ff00ff000100, 0x0101ff00ff000101, 0x0101ff0000ff0001,+ 0x0101ff0000ff0100, 0x0101ff000000ff00, 0x0101ff0000000000, 0x0101ff00000001ff,+ 0x0101ff0000000101, 0x0101ff000001ff00, 0x0101ff00000100ff, 0x0101ff0001ff0000,+ 0x0101ff000100ffff, 0x0101ff000100ff01, 0x0101ff0001000001, 0x0101ff0001000100,+ 0x0101ff01ffffff01, 0x0101ff01ffff01ff, 0x0101ff01ffff0101, 0x0101ff01ff00ffff,+ 0x0101ff01ff000100, 0x0101ff01ff01ff01, 0x0101ff01ff0101ff, 0x0101ff01ff010101,+ 0x0101ff0100ff0000, 0x0101ff010000ff00, 0x0101ff0100000001, 0x0101ff0100000100,+ 0x0101ff0100010000, 0x0101ff0101ffffff, 0x0101ff0101ffff01, 0x0101ff0101ff01ff,+ 0x0101ff0101ff0101, 0x0101ff0101000000, 0x0101ff010101ffff, 0x0101ff010101ff01,+ 0x0101ff01010101ff, 0x0101ff0101010101, 0x010100ffff000100, 0x010100ffff010000,+ 0x010100ff00ffff00, 0x010100ff00ff00ff, 0x010100ff0000ffff, 0x010100ff000000ff,+ 0x010100ff00000000, 0x010100ff000001ff, 0x010100ff00000101, 0x010100ff0001ff00,+ 0x010100ff00010000, 0x010100ff00010001, 0x010100ff000101ff, 0x010100ff00010100,+ 0x010100ff01ff0000, 0x01010000ffff0001, 0x01010000ffff0100, 0x01010000ff00ffff,+ 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, 0x01010000ff010001,+ 0x01010000ff010100, 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00,+ 0x01010000000000ff, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100,+ 0x0101000000010000, 0x0101000000010101, 0x0101000001ffff00, 0x0101000001ff00ff,+ 0x0101000001ff0000, 0x0101000001ff0001, 0x0101000001ff0100, 0x010100000100ff01,+ 0x0101000001000000, 0x01010000010001ff, 0x01010001ffff0000, 0x01010001ff00ff00,+ 0x01010001ff000001, 0x01010001ff000101, 0x01010001ff01ff00, 0x01010001ff010000,+ 0x0101000100ff00ff, 0x0101000100ff0001, 0x0101000100ff0101, 0x010100010000ff01,+ 0x0101000100000000, 0x0101000100000001, 0x01010001000001ff, 0x010100010001ffff,+ 0x010100010001ff01, 0x0101000101ff0001, 0x010100010100ffff, 0x0101000101000000,+ 0x0101000101000001, 0x0101000101000100, 0x010100010101ff00, 0x01010001010100ff,+ 0x0101000101010001, 0x010101ffffffffff, 0x010101ffffffff01, 0x010101ffffff01ff,+ 0x010101ffffff0101, 0x010101ffff01ffff, 0x010101ffff01ff01, 0x010101ffff0101ff,+ 0x010101ffff010101, 0x010101ff0000ff00, 0x010101ff000000ff, 0x010101ff00000001,+ 0x010101ff00000100, 0x010101ff01ffffff, 0x010101ff01ffff01, 0x010101ff01ff01ff,+ 0x010101ff01ff0101, 0x010101ff01000000, 0x010101ff0101ffff, 0x010101ff0101ff01,+ 0x010101ff010101ff, 0x010101ff01010101, 0x01010100ffff0000, 0x01010100ff0000ff,+ 0x01010100ff000100, 0x01010100ff01ff00, 0x01010100ff010000, 0x0101010000ffff00,+ 0x010101000000ffff, 0x0101010000000000, 0x0101010000000101, 0x010101000001ff00,+ 0x0101010000010001, 0x0101010000010100, 0x010101000100ffff, 0x0101010001000001,+ 0x01010101ffffffff, 0x01010101ffffff01, 0x01010101ffff01ff, 0x01010101ffff0101,+ 0x01010101ff01ffff, 0x01010101ff01ff01, 0x01010101ff0101ff, 0x01010101ff010101,+ 0x010101010000ff00, 0x01010101000000ff, 0x0101010100000001, 0x0101010101ffffff,+ 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000,+ 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101,+GGML_TABLE_END()+#else+GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S)+ 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000,+ 0x00020002, 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101,+ 0x02000000, 0x02000002, 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200,+ 0x02020202, 0x00000110, 0x00000111, 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212,+ 0x00020111, 0x01000011, 0x01000112, 0x01000211, 0x01010012, 0x01010111, 0x01010212, 0x01020011,+ 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011, 0x02010110, 0x02010112, 0x02020111,+ 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020, 0x00020022, 0x00020220,+ 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020, 0x02000022,+ 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,+ 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101,+ 0x01011202, 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110,+ 0x00001111, 0x00001112, 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111,+ 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010,+ 0x01021012, 0x01021111, 0x01021210, 0x01021212, 0x02001011, 0x02011011, 0x02011111, 0x02011210,+ 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112, 0x02021211, 0x00011120, 0x00011221,+ 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220, 0x01021020, 0x01021021,+ 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000, 0x00002002,+ 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,+ 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101,+ 0x02022000, 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211,+ 0x00022110, 0x00022111, 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110,+ 0x01022211, 0x02012011, 0x02012110, 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022,+ 0x00002220, 0x00002222, 0x00012121, 0x00022020, 0x00022022, 0x00022220, 0x00022222, 0x01002121,+ 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020, 0x02002022, 0x02002121, 0x02002220,+ 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222, 0x00110000, 0x00110001,+ 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000, 0x01110101,+ 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,+ 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012,+ 0x00110111, 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010,+ 0x01110011, 0x01110012, 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111,+ 0x02100110, 0x02110012, 0x02110111, 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122,+ 0x00120121, 0x01100020, 0x01100122, 0x01100221, 0x01110022, 0x01110121, 0x01110220, 0x01110222,+ 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120, 0x02110122, 0x02120121, 0x00101001,+ 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201, 0x00121001, 0x00121102,+ 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100, 0x01111101,+ 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,+ 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101,+ 0x02121201, 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112,+ 0x00111211, 0x00121010, 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110,+ 0x01101111, 0x01101112, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211,+ 0x01111212, 0x01121011, 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101010, 0x02101012,+ 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010, 0x02111011, 0x02111110, 0x02111111,+ 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, 0x00101021, 0x00101120,+ 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021, 0x00121122,+ 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,+ 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221,+ 0x01121222, 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001,+ 0x00112102, 0x00122101, 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101,+ 0x01112200, 0x01112202, 0x01122000, 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101,+ 0x02112001, 0x02112100, 0x02122101, 0x00112010, 0x00112012, 0x00112111, 0x00112212, 0x00122011,+ 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111,+ 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212, 0x02102211, 0x02112011,+ 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221, 0x00112122,+ 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,+ 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222,+ 0x00200000, 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101,+ 0x00220200, 0x00220202, 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000,+ 0x02200002, 0x02200200, 0x02200202, 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200,+ 0x02220202, 0x00200111, 0x00210011, 0x00210110, 0x00210211, 0x00220111, 0x01200012, 0x01200110,+ 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011, 0x01220110, 0x01220111, 0x01220112,+ 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021, 0x00200220, 0x00200222,+ 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121, 0x01210021,+ 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,+ 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201,+ 0x00221101, 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200,+ 0x01211202, 0x01221102, 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101,+ 0x00201211, 0x00211111, 0x00221011, 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011,+ 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x01221210, 0x02201211, 0x02211010,+ 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011, 0x02221110, 0x02221112, 0x02221211,+ 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021, 0x01201221, 0x01211121,+ 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222, 0x00202000,+ 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,+ 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202,+ 0x02222000, 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211,+ 0x00222111, 0x01202112, 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112,+ 0x01222211, 0x02202111, 0x02212010, 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020,+ 0x00202022, 0x00202220, 0x00202222, 0x00222020, 0x00222022, 0x00222220, 0x00222222, 0x01202121,+ 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020, 0x02202022, 0x02202220, 0x02202222,+ 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101, 0x10010001, 0x10010102,+ 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001, 0x11020100,+ 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,+ 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011,+ 0x10020112, 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111,+ 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110,+ 0x12000112, 0x12010010, 0x12010012, 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121,+ 0x10010021, 0x10010120, 0x10010122, 0x10020121, 0x11000021, 0x11010022, 0x11010121, 0x11010222,+ 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121, 0x10001001, 0x10011101, 0x10011201,+ 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, 0x11011102,+ 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102, 0x12001201,+ 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,+ 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010,+ 0x10021111, 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010,+ 0x11011011, 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110,+ 0x11021111, 0x11021112, 0x11021211, 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011,+ 0x12011110, 0x12011111, 0x12011112, 0x12011211, 0x12011212, 0x12021111, 0x12021210, 0x12021212,+ 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121, 0x10011220, 0x10011222, 0x10021021,+ 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220, 0x11011020, 0x11011021,+ 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220, 0x12001021,+ 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,+ 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101,+ 0x11012200, 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100,+ 0x12012102, 0x12012201, 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010,+ 0x10012110, 0x10012111, 0x10012210, 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111,+ 0x11002212, 0x11012011, 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010,+ 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112, 0x12002211, 0x12012012, 0x12012111,+ 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211, 0x10012122, 0x11002120,+ 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221, 0x12012120,+ 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,+ 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001,+ 0x11110100, 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201,+ 0x12110101, 0x12110200, 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210,+ 0x10100211, 0x10100212, 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211,+ 0x10120010, 0x10120111, 0x10120112, 0x10120210, 0x10120212, 0x11100011, 0x11100110, 0x11100111,+ 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, 0x11110112,+ 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, 0x11120211,+ 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120010,+ 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,+ 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122,+ 0x11110221, 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221,+ 0x12110222, 0x12120120, 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102,+ 0x10111200, 0x10111201, 0x10121001, 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100,+ 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, 0x11111001, 0x11111100, 0x11111101,+ 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, 0x11121002, 0x11121100, 0x11121101,+ 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001, 0x12111100, 0x12111101,+ 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012,+ 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,+ 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112,+ 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210,+ 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210,+ 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210,+ 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010,+ 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110,+ 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022, 0x10101120, 0x10101122,+ 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121020,+ 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,+ 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022,+ 0x11111120, 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120,+ 0x11121121, 0x11121221, 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222,+ 0x12111021, 0x12111121, 0x12111222, 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221,+ 0x10102100, 0x10102101, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10112200, 0x10122001,+ 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001, 0x11112100, 0x11112101, 0x11112102,+ 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101, 0x12102002, 0x12102201,+ 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011, 0x10102012,+ 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,+ 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012,+ 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110,+ 0x11122111, 0x11122112, 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110,+ 0x12112111, 0x12112112, 0x12112210, 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121,+ 0x10112222, 0x10122020, 0x10122121, 0x10122122, 0x10122221, 0x11102121, 0x11102220, 0x11102221,+ 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221, 0x11122022, 0x11122121, 0x11122220,+ 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122, 0x12112220, 0x12112222,+ 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100, 0x11210000,+ 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,+ 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012,+ 0x10210111, 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011,+ 0x11210111, 0x11210112, 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212,+ 0x12210012, 0x12210111, 0x12220011, 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221,+ 0x11200020, 0x11200021, 0x11200122, 0x11210121, 0x11210122, 0x11210220, 0x11220020, 0x12200121,+ 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002, 0x10211101, 0x10211102, 0x10211202,+ 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101, 0x11201200, 0x11201202,+ 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000, 0x11221002,+ 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,+ 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210,+ 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112,+ 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011,+ 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011,+ 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221112, 0x12221210,+ 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122, 0x10221220, 0x10221221, 0x11201020,+ 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220,+ 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121, 0x12201222,+ 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,+ 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001,+ 0x11222201, 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010,+ 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111,+ 0x11202112, 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010,+ 0x11222111, 0x11222212, 0x12202012, 0x12202110, 0x12202212, 0x12212111, 0x12222011, 0x12222110,+ 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220, 0x11202021, 0x11202120, 0x11202221,+ 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121, 0x11222221, 0x12202122,+ 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200, 0x20000202,+ 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,+ 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101,+ 0x22020000, 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112,+ 0x20010211, 0x20020111, 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111,+ 0x21010112, 0x21010210, 0x21010211, 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211,+ 0x22010110, 0x22010112, 0x22010211, 0x22020111, 0x20000020, 0x20000022, 0x20000220, 0x20000222,+ 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222, 0x21010021, 0x21010120, 0x21010221,+ 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121, 0x22020020, 0x22020022,+ 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001, 0x21011101,+ 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,+ 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111,+ 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111,+ 0x21021112, 0x21021210, 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010,+ 0x22011012, 0x22011111, 0x22011210, 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121,+ 0x21001021, 0x21001120, 0x21001221, 0x21001222, 0x21011020, 0x21011121, 0x21011221, 0x21011222,+ 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021, 0x22011222, 0x22021120, 0x20002000,+ 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002, 0x20022200, 0x20022202,+ 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201, 0x22002000,+ 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,+ 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110,+ 0x21002112, 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110,+ 0x22002111, 0x22012112, 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222,+ 0x20012121, 0x20022020, 0x20022022, 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120,+ 0x21012122, 0x22002020, 0x22002022, 0x22002220, 0x22002222, 0x22012121, 0x22022020, 0x22022022,+ 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102, 0x20110200, 0x20110201, 0x20120101,+ 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202, 0x21120201, 0x21120202,+ 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011, 0x20100110,+ 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,+ 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111,+ 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111,+ 0x22110210, 0x22120011, 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120,+ 0x20110221, 0x20120121, 0x21100120, 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121,+ 0x21110220, 0x21120122, 0x21120221, 0x22100121, 0x22110120, 0x22110122, 0x22120221, 0x20101001,+ 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200, 0x20121102, 0x21101000, 0x21101202,+ 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201, 0x21121000, 0x21121001,+ 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101, 0x22111200,+ 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,+ 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212,+ 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012,+ 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110,+ 0x21121111, 0x21121112, 0x21121211, 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012,+ 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121012, 0x22121111,+ 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020, 0x20111121, 0x20111221, 0x20121020,+ 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021, 0x21111022, 0x21111121,+ 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221, 0x22101222,+ 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,+ 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102,+ 0x21112202, 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101,+ 0x20102110, 0x20102112, 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212,+ 0x20122010, 0x20122011, 0x20122110, 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210,+ 0x21102212, 0x21112011, 0x21112110, 0x21112111, 0x21112112, 0x21112211, 0x21122012, 0x21122111,+ 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010, 0x22112012, 0x22112111, 0x22112212,+ 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120, 0x21102122, 0x21102221,+ 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120, 0x22112121,+ 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,+ 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000,+ 0x22200002, 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202,+ 0x20200111, 0x20200211, 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112,+ 0x21200211, 0x21210011, 0x21210111, 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111,+ 0x22210010, 0x22210012, 0x22210112, 0x22210211, 0x20200022, 0x20200220, 0x20200222, 0x20210020,+ 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121, 0x21210021, 0x21210122, 0x21210221,+ 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121, 0x22220020, 0x22220022,+ 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000, 0x21211100,+ 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,+ 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112,+ 0x20221211, 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211,+ 0x21221111, 0x21221212, 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012,+ 0x22211111, 0x22211210, 0x20201121, 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121,+ 0x21201120, 0x21201122, 0x21201222, 0x21211022, 0x21211121, 0x21211122, 0x21211220, 0x21221020,+ 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122, 0x22211221, 0x22221021, 0x22221120,+ 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000, 0x20222002, 0x20222200,+ 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002, 0x22202200,+ 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,+ 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011,+ 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222,+ 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020,+ 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,+GGML_TABLE_END()+#endif++#endif // GGML_COMMON_IMPL+#endif // GGML_COMMON_IMPLdiff --git a/python/freetoken/kernel/csrc/ggml/src/ggml-impl.h b/python/freetoken/kernel/csrc/ggml/src/ggml-impl.hnew file mode 100644index 00000000..ae26e0c2--- /dev/null+++ b/python/freetoken/kernel/csrc/ggml/src/ggml-impl.h@@ -0,0 +1,795 @@+#pragma once++// GGML internal header++#include "ggml.h"+#include "gguf.h"++#include <assert.h>+#include <math.h>+#include <stdlib.h> // load `stdlib.h` before other headers to work around MinGW bug: https://sourceforge.net/p/mingw-w64/bugs/192/+#include <stdbool.h>+#include <stdint.h>+#include <string.h>++#ifdef __ARM_FEATURE_SVE+#include <arm_sve.h>+#endif // __ARM_FEATURE_SVE++#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__)+// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:+//+// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/+//+#include <arm_neon.h>+#endif++#ifdef __cplusplus+extern "C" {+#endif++void ggml_print_backtrace(void);++uint64_t ggml_graph_next_uid(void);++#ifndef MIN+# define MIN(a, b) ((a) < (b) ? (a) : (b))+#endif++#ifndef MAX+# define MAX(a, b) ((a) > (b) ? (a) : (b))+#endif++// required for mmap as gguf only guarantees 32-byte alignment+#define TENSOR_ALIGNMENT 32++// static_assert should be a #define, but if it's not,+// fall back to the _Static_assert C11 keyword.+// if C99 - static_assert is noop+// ref: https://stackoverflow.com/a/53923785/4039976+#ifndef __cplusplus+ #ifndef static_assert+ #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201100L)+ #define static_assert(cond, msg) _Static_assert(cond, msg)+ #else+ #define static_assert(cond, msg) struct global_scope_noop_trick+ #endif+ #endif+#endif++static inline int ggml_up32(int n) {+ return (n + 31) & ~31;+}++//static inline int ggml_up64(int n) {+// return (n + 63) & ~63;+//}++static inline int ggml_up(int n, int m) {+ // assert m is a power of 2+ GGML_ASSERT((m & (m - 1)) == 0);+ return (n + m - 1) & ~(m - 1);+}++// TODO: move to ggml.h? (won't be able to inline)+static bool ggml_are_same_layout(const struct ggml_tensor * a, const struct ggml_tensor * b) {+ if (a->type != b->type) {+ return false;+ }+ for (int i = 0; i < GGML_MAX_DIMS; i++) {+ if (a->ne[i] != b->ne[i]) {+ return false;+ }+ if (a->nb[i] != b->nb[i]) {+ return false;+ }+ }+ return true;+}++static bool ggml_op_is_empty(enum ggml_op op) {+ switch (op) {+ case GGML_OP_NONE:+ case GGML_OP_RESHAPE:+ case GGML_OP_TRANSPOSE:+ case GGML_OP_VIEW:+ case GGML_OP_PERMUTE:+ return true;+ default:+ return false;+ }+}++static inline bool ggml_impl_is_view(const struct ggml_tensor * t) {+ return t->view_src != NULL;+}++static inline float ggml_compute_softplus_f32(float input) {+ return (input > 20.0f) ? input : logf(1 + expf(input));+}+//+// logging+//++GGML_ATTRIBUTE_FORMAT(2, 3)+GGML_API void ggml_log_internal (enum ggml_log_level level, const char * format, ...);+GGML_API void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data);++#define GGML_LOG(...) ggml_log_internal(GGML_LOG_LEVEL_NONE , __VA_ARGS__)+#define GGML_LOG_INFO(...) ggml_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)+#define GGML_LOG_WARN(...) ggml_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)+#define GGML_LOG_ERROR(...) ggml_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)+#define GGML_LOG_DEBUG(...) ggml_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)+#define GGML_LOG_CONT(...) ggml_log_internal(GGML_LOG_LEVEL_CONT , __VA_ARGS__)++#define GGML_DEBUG 0++#if (GGML_DEBUG >= 1)+#define GGML_PRINT_DEBUG(...) GGML_LOG_DEBUG(__VA_ARGS__)+#else+#define GGML_PRINT_DEBUG(...)+#endif++#if (GGML_DEBUG >= 5)+#define GGML_PRINT_DEBUG_5(...) GGML_LOG_DEBUG(__VA_ARGS__)+#else+#define GGML_PRINT_DEBUG_5(...)+#endif++#if (GGML_DEBUG >= 10)+#define GGML_PRINT_DEBUG_10(...) GGML_LOG_DEBUG(__VA_ARGS__)+#else+#define GGML_PRINT_DEBUG_10(...)+#endif++// tensor params++static void ggml_set_op_params(struct ggml_tensor * tensor, const void * params, size_t params_size) {+ GGML_ASSERT(tensor != NULL); // silence -Warray-bounds warnings+ assert(params_size <= GGML_MAX_OP_PARAMS);+ memcpy(tensor->op_params, params, params_size);+}++static int32_t ggml_get_op_params_i32(const struct ggml_tensor * tensor, uint32_t i) {+ assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));+ return ((const int32_t *)(tensor->op_params))[i];+}++static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t i) {+ assert(i < GGML_MAX_OP_PARAMS / sizeof(float));+ return ((const float *)(tensor->op_params))[i];+}++// [TAG_GGML_PREC]+// - GGML_OP_MUL_MAT+// 0 - acc+// 1 - hint+// 2 - src0 precision+// 3 - src1 precision+//+// - GGML_OP_MUL_MAT_ID+// 0 - acc+// 1 - hint+// 2 - src0 precision+// 3 - src1 precision+static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {+ assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));+ ((int32_t *)(tensor->op_params))[i] = value;+}++static void ggml_set_op_params_f32(struct ggml_tensor * tensor, uint32_t i, float value) {+ assert(i < GGML_MAX_OP_PARAMS / sizeof(float));+ ((float *)(tensor->op_params))[i] = value;+}++struct ggml_map_custom1_op_params {+ ggml_custom1_op_t fun;+ int n_tasks;+ void * userdata;+};++struct ggml_map_custom2_op_params {+ ggml_custom2_op_t fun;+ int n_tasks;+ void * userdata;+};++struct ggml_map_custom3_op_params {+ ggml_custom3_op_t fun;+ int n_tasks;+ void * userdata;+};++struct ggml_custom_op_params {+ ggml_custom_op_t fun;+ int n_tasks;+ void * userdata;+};++// bitset++typedef uint32_t ggml_bitset_t;++static_assert(sizeof(ggml_bitset_t) == 4, "bitset_t constants must be updated");+#define BITSET_SHR 5 // log2(sizeof(ggml_bitset_t)*8)+#define BITSET_MASK (sizeof(ggml_bitset_t)*8 - 1)++static size_t ggml_bitset_size(size_t n) {+ return (n + BITSET_MASK) >> BITSET_SHR;+}++static inline bool ggml_bitset_get(const ggml_bitset_t * bitset, size_t i) {+ return !!(bitset[i >> BITSET_SHR] & (1u << (i & BITSET_MASK)));+}++static inline void ggml_bitset_set(ggml_bitset_t * bitset, size_t i) {+ bitset[i >> BITSET_SHR] |= (1u << (i & BITSET_MASK));+}++static inline void ggml_bitset_clear(ggml_bitset_t * bitset, size_t i) {+ bitset[i >> BITSET_SHR] &= ~(1u << (i & BITSET_MASK));+}++// hash set++#define GGML_HASHSET_FULL ((size_t)-1)+#define GGML_HASHSET_ALREADY_EXISTS ((size_t)-2)++struct ggml_hash_set {+ size_t size;+ ggml_bitset_t * used; // whether or not the keys are in use i.e. set+ struct ggml_tensor ** keys; // actual tensors in the set, keys[i] is only defined if ggml_bitset_get(used, i)+};++struct ggml_hash_set ggml_hash_set_new(size_t size);+void ggml_hash_set_free(struct ggml_hash_set * hash_set);++// returns the minimum size for a hash set that can hold min_sz elements+size_t ggml_hash_size(size_t min_sz);++// remove all elements from the hash set+void ggml_hash_set_reset(struct ggml_hash_set * hash_set);++// returns true if key is in the hash set+static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key);++// returns GGML_HASHSET_FULL if table is full, otherwise the current index of the key or where it should be inserted+static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key);++// returns GGML_HASHSET_ALREADY_EXISTS if key already exists, index otherwise, asserts if table is full+static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);++// return index, asserts if table is full+static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key);++// hash function for ggml_tensor+static inline size_t ggml_hash(const struct ggml_tensor * p) {+ // the last 4 bits are always zero due to alignment+ return (size_t)(uintptr_t)p >> 4;+}++static size_t ggml_hash_find(const struct ggml_hash_set * hash_set, const struct ggml_tensor * key) {+ size_t h = ggml_hash(key) % hash_set->size;++ // linear probing+ size_t i = h;+ while (ggml_bitset_get(hash_set->used, i) && hash_set->keys[i] != key) {+ i = (i + 1) % hash_set->size;+ if (i == h) {+ // visited all hash table entries -> not found+ return GGML_HASHSET_FULL;+ }+ }+ return i;+}++static bool ggml_hash_contains(const struct ggml_hash_set * hash_set, struct ggml_tensor * key) {+ size_t i = ggml_hash_find(hash_set, key);+ return i != GGML_HASHSET_FULL && ggml_bitset_get(hash_set->used, i);+}++static size_t ggml_hash_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {+ size_t h = ggml_hash(key) % hash_set->size;++ // linear probing+ size_t i = h;+ do {+ if (!ggml_bitset_get(hash_set->used, i)) {+ ggml_bitset_set(hash_set->used, i);+ hash_set->keys[i] = key;+ return i;+ }+ if (hash_set->keys[i] == key) {+ return GGML_HASHSET_ALREADY_EXISTS;+ }+ i = (i + 1) % hash_set->size;+ } while (i != h);++ // visited all hash table entries -> not found+ GGML_ABORT("fatal error");+}++static size_t ggml_hash_find_or_insert(struct ggml_hash_set * hash_set, struct ggml_tensor * key) {+ size_t h = ggml_hash(key) % hash_set->size;++ // linear probing+ size_t i = h;+ do {+ if (!ggml_bitset_get(hash_set->used, i)) {+ ggml_bitset_set(hash_set->used, i);+ hash_set->keys[i] = key;+ return i;+ }+ if (hash_set->keys[i] == key) {+ return i;+ }+ i = (i + 1) % hash_set->size;+ } while (i != h);++ // visited all hash table entries -> not found+ GGML_ABORT("fatal error");+}++// computation graph++enum ggml_cgraph_eval_order {+ GGML_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,+ GGML_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,+ GGML_CGRAPH_EVAL_ORDER_COUNT+};++struct ggml_cgraph {+ int size; // maximum number of nodes/leafs/grads/grad_accs+ int n_nodes; // number of nodes currently in use+ int n_leafs; // number of leafs currently in use++ struct ggml_tensor ** nodes; // tensors with data that can change if the graph is evaluated+ struct ggml_tensor ** grads; // the outputs of these tensors are the gradients of the nodes+ struct ggml_tensor ** grad_accs; // accumulators for node gradients+ struct ggml_tensor ** leafs; // tensors with constant data+ int32_t * use_counts;// number of uses of each tensor, indexed by hash table slot++ struct ggml_hash_set visited_hash_set;++ enum ggml_cgraph_eval_order order;++ // an optional identifier that can be utilized to recognize same graphs if two non-zero values match+ // a value of 0 means it is not set and should be ignored+ uint64_t uid;+};++// returns a slice of cgraph with nodes [i0, i1)+// the slice does not have leafs or gradients+// if you need the gradients, get them from the original graph+struct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph, int i0, int i1);++// ggml-alloc.c: true if the operation can reuse memory from its sources+GGML_API bool ggml_op_can_inplace(enum ggml_op op);+++// Memory allocation++GGML_API void * ggml_aligned_malloc(size_t size);+GGML_API void ggml_aligned_free(void * ptr, size_t size);++// FP16 <-> FP32+// ref: https://github.com/Maratyszcza/FP16++static inline float fp32_from_bits(uint32_t w) {+ union {+ uint32_t as_bits;+ float as_value;+ } fp32;+ fp32.as_bits = w;+ return fp32.as_value;+}++static inline uint32_t fp32_to_bits(float f) {+ union {+ float as_value;+ uint32_t as_bits;+ } fp32;+ fp32.as_value = f;+ return fp32.as_bits;+}++static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {+ const uint32_t w = (uint32_t) h << 16;+ const uint32_t sign = w & UINT32_C(0x80000000);+ const uint32_t two_w = w + w;++ const uint32_t exp_offset = UINT32_C(0xE0) << 23;+#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)+ const float exp_scale = 0x1.0p-112f;+#else+ const float exp_scale = fp32_from_bits(UINT32_C(0x7800000));+#endif+ const float normalized_value = fp32_from_bits((two_w >> 4) + exp_offset) * exp_scale;++ const uint32_t magic_mask = UINT32_C(126) << 23;+ const float magic_bias = 0.5f;+ const float denormalized_value = fp32_from_bits((two_w >> 17) | magic_mask) - magic_bias;++ const uint32_t denormalized_cutoff = UINT32_C(1) << 27;+ const uint32_t result = sign |+ (two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));+ return fp32_from_bits(result);+}++static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {+#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || defined(__GNUC__) && !defined(__STRICT_ANSI__)) && (!defined(__cplusplus) || __cplusplus >= 201703L)+ const float scale_to_inf = 0x1.0p+112f;+ const float scale_to_zero = 0x1.0p-110f;+#else+ const float scale_to_inf = fp32_from_bits(UINT32_C(0x77800000));+ const float scale_to_zero = fp32_from_bits(UINT32_C(0x08800000));+#endif+ float base = (fabsf(f) * scale_to_inf) * scale_to_zero;++ const uint32_t w = fp32_to_bits(f);+ const uint32_t shl1_w = w + w;+ const uint32_t sign = w & UINT32_C(0x80000000);+ uint32_t bias = shl1_w & UINT32_C(0xFF000000);+ if (bias < UINT32_C(0x71000000)) {+ bias = UINT32_C(0x71000000);+ }++ base = fp32_from_bits((bias >> 1) + UINT32_C(0x07800000)) + base;+ const uint32_t bits = fp32_to_bits(base);+ const uint32_t exp_bits = (bits >> 13) & UINT32_C(0x00007C00);+ const uint32_t mantissa_bits = bits & UINT32_C(0x00000FFF);+ const uint32_t nonsign = exp_bits + mantissa_bits;+ return (sign >> 16) | (shl1_w > UINT32_C(0xFF000000) ? UINT16_C(0x7E00) : nonsign);+}++#define GGML_COMPUTE_FP16_TO_FP32(x) ggml_compute_fp16_to_fp32(x)+#define GGML_COMPUTE_FP32_TO_FP16(x) ggml_compute_fp32_to_fp16(x)++#define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x)+#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)++static inline float ggml_e8m0_to_fp32(uint8_t x) {+ uint32_t bits; // Stores the raw bit representation of the float++ // Handle special case for minimum exponent (denormalized float)+ if (x == 0) {+ // Bit pattern for 2^(-127):+ // - Sign bit: 0 (positive)+ // - Exponent: 0 (denormalized number)+ // - Mantissa: 0x400000 (0.5 in fractional form)+ // Value = 0.5 * 2^(-126) = 2^(-127)+ bits = 0x00400000;+ }+ // note: disabled as we don't need to handle NaNs+ //// Handle special case for NaN (all bits set)+ //else if (x == 0xFF) {+ // // Standard quiet NaN pattern:+ // // - Sign bit: 0+ // // - Exponent: all 1s (0xFF)+ // // - Mantissa: 0x400000 (quiet NaN flag)+ // bits = 0x7FC00000;+ //}+ // Normalized values (most common case)+ else {+ // Construct normalized float by shifting exponent into position:+ // - Exponent field: 8 bits (positions 30-23)+ // - Mantissa: 0 (implicit leading 1)+ // Value = 2^(x - 127)+ bits = (uint32_t) x << 23;+ }++ float result; // Final float value+ // Safely reinterpret bit pattern as float without type-punning issues+ memcpy(&result, &bits, sizeof(float));+ return result;+}++// Equal to ggml_e8m0_to_fp32/2+// Useful with MXFP4 quantization since the E0M2 values are doubled+static inline float ggml_e8m0_to_fp32_half(uint8_t x) {+ uint32_t bits;++ // For x < 2: use precomputed denormal patterns+ if (x < 2) {+ // 0x00200000 = 2^(-128), 0x00400000 = 2^(-127)+ bits = 0x00200000 << x;+ }+ // For x >= 2: normalized exponent adjustment+ else {+ // 0.5 * 2^(x-127) = 2^(x-128) = normalized with exponent (x-1)+ bits = (uint32_t)(x - 1) << 23;+ }+ // Note: NaNs are not handled here++ float result;+ memcpy(&result, &bits, sizeof(float));+ return result;+}++#define GGML_E8M0_TO_FP32(x) ggml_e8m0_to_fp32(x)+#define GGML_E8M0_TO_FP32_HALF(x) ggml_e8m0_to_fp32_half(x)++// UE4M3: unsigned, 4 exp bits (bias=7), 3 mantissa bits+// Returns value * 0.5 to match kvalues_mxfp4 convention (kvalues = 2 * E2M1_float)+static inline float ggml_ue4m3_to_fp32(uint8_t x) {+ if (x == 0 || x == 0x7F) {+ return 0.0f;+ }+ int exp = (x >> 3) & 0xF;+ int man = x & 0x7;+ float raw;+ if (exp == 0) {+ raw = ldexpf((float) man, -9);+ } else {+ raw = ldexpf(1.0f + (float) man / 8.0f, exp - 7);+ }+ return raw * 0.5f;+}++static inline uint8_t ggml_fp32_to_ue4m3(float x) {+ if (!(x > 0.0f)) {+ return 0;+ }+ if (x > 448.0f) {+ x = 448.0f;+ }+ uint32_t bits;+ memcpy(&bits, &x, 4);+ int fp32_exp = ((bits >> 23) & 0xFF) - 127;+ int fp32_man = (bits >> 20) & 0x7;+ int ue4m3_exp = fp32_exp + 7;+ if (ue4m3_exp <= 0) {+ // subnormal: value = man * 2^-9, man = round(x * 2^9)+ int man = (int) (x * 512.0f + 0.5f);+ if (man > 7) {+ man = 7;+ }+ if (man < 1) {+ return 0;+ }+ return (uint8_t) man;+ }+ if (ue4m3_exp >= 15) {+ return 0x7E;+ }+ int round_bit = (bits >> 19) & 1;+ int ue4m3_man = fp32_man + round_bit;+ if (ue4m3_man > 7) {+ ue4m3_man = 0;+ ue4m3_exp++;+ if (ue4m3_exp >= 15) {+ return 0x7E;+ }+ }+ return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man);+}++/**+ * Converts brain16 to float32.+ *+ * The bfloat16 floating point format has the following structure:+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌──┴───┐┌─┴───┐+ * 0b0000000000000000 brain16+ *+ * Since bf16 has the same number of exponent bits as a 32bit float,+ * encoding and decoding numbers becomes relatively straightforward.+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌──┴───┐┌─┴───────────────────┐+ * 0b00000000000000000000000000000000 IEEE binary32+ *+ * For comparison, the standard fp16 format has fewer exponent bits.+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌─┴─┐┌─┴──────┐+ * 0b0000000000000000 IEEE binary16+ *+ * @see IEEE 754-2008+ */+static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {+ union {+ float f;+ uint32_t i;+ } u;+ u.i = (uint32_t)h.bits << 16;+ return u.f;+}++/**+ * Converts float32 to brain16.+ *+ * This is binary identical with Google Brain float conversion.+ * Floats shall round to nearest even, and NANs shall be quiet.+ * Subnormals aren't flushed to zero, except perhaps when used.+ * This code should vectorize nicely if using modern compilers.+ */+static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {+ ggml_bf16_t h;+ union {+ float f;+ uint32_t i;+ } u;+ u.f = s;+ if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */+ h.bits = (u.i >> 16) | 64; /* force to quiet */+ return h;+ }+ h.bits = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;+ return h;+}++#define GGML_FP32_TO_BF16(x) ggml_compute_fp32_to_bf16(x)+#define GGML_BF16_TO_FP32(x) ggml_compute_bf16_to_fp32(x)++static inline int32_t ggml_node_get_use_count(const struct ggml_cgraph * cgraph, int node_idx) {+ const struct ggml_tensor * node = cgraph->nodes[node_idx];++ size_t hash_pos = ggml_hash_find(&cgraph->visited_hash_set, node);+ if (!ggml_bitset_get(cgraph->visited_hash_set.used, hash_pos)) {+ return 0;+ }+ return cgraph->use_counts[hash_pos];+}++// return true if the node's results are only used by N other nodes+// and can be fused into their calculations.+static inline bool ggml_node_has_n_uses(const struct ggml_cgraph * cgraph, int node_idx, int32_t n_uses) {+ const struct ggml_tensor * node = cgraph->nodes[node_idx];++ // check the use count against how many we're replacing+ if (ggml_node_get_use_count(cgraph, node_idx) != n_uses) {+ return false;+ }++ // if node is a view, some other node might be using the intermediate result+ // via the view source.+ if (node->view_src) {+ return false;+ }++ // If the user requested output for the node, can't fuse+ if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {+ return false;+ }++ return true;+}++// Returns true if nodes with indices { node_idxs } are the sequence of ggml_ops in ops[]+// and are fusable. Nodes are considered fusable according to this function if:+// - all nodes except the last have only one use and are not views/outputs (see ggml_node_has_N_uses).+// - all nodes except the last are a src of the following node.+// - all nodes are the same shape.+// TODO: Consider allowing GGML_OP_NONE nodes in between+static inline bool ggml_can_fuse_ext(const struct ggml_cgraph * cgraph, const int * node_idxs, const enum ggml_op * ops, int num_ops) {+ for (int i = 0; i < num_ops; ++i) {+ if (node_idxs[i] >= cgraph->n_nodes) {+ return false;+ }++ struct ggml_tensor * node = cgraph->nodes[node_idxs[i]];+ if (node->op != ops[i]) {+ return false;+ }+ if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {+ return false;+ }+ if (i < num_ops - 1 && !ggml_node_has_n_uses(cgraph, node_idxs[i], 1)) {+ return false;+ }+ if (i > 0) {+ struct ggml_tensor * prev = cgraph->nodes[node_idxs[i - 1]];+ if (node->src[0] != prev && node->src[1] != prev) {+ return false;+ }+ if (!ggml_are_same_shape(node, prev)) {+ return false;+ }+ }+ }+ return true;+}++// same as above, for sequential indices starting at node_idx+static inline bool ggml_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, const enum ggml_op * ops, int num_ops) {+ assert(num_ops < 32);++ if (node_idx + num_ops > cgraph->n_nodes) {+ return false;+ }++ int idxs[32];+ for (int i = 0; i < num_ops; ++i) {+ idxs[i] = node_idx + i;+ }++ return ggml_can_fuse_ext(cgraph, idxs, ops, num_ops);+}++GGML_API bool ggml_can_fuse_subgraph_ext(const struct ggml_cgraph * cgraph,+ const int * node_idxs,+ int count,+ const enum ggml_op * ops,+ const int * outputs,+ int num_outputs);++// Returns true if the subgraph formed by {node_idxs} can be fused+// checks whethers all nodes which are not part of outputs can be elided+// by checking if their num_uses are confined to the subgraph+static inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph,+ int node_idx,+ int count,+ const enum ggml_op * ops,+ const int * outputs,+ int num_outputs) {+ GGML_ASSERT(count < 32);+ if (node_idx + count > cgraph->n_nodes) {+ return false;+ }++ int idxs[32];++ for (int i = 0; i < count; ++i) {+ idxs[i] = node_idx + i;+ }++ return ggml_can_fuse_subgraph_ext(cgraph, idxs, count, ops, outputs, num_outputs);+}++#ifdef __cplusplus+}+#endif++#ifdef __cplusplus+#include <array>+#include <initializer_list>+#include <vector>++// nicer C++ syntax for ggml_can_fuse+inline bool ggml_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {+ return ggml_can_fuse(cgraph, node_idx, ops.begin(), (int)ops.size());+}++inline bool ggml_can_fuse_subgraph(const struct ggml_cgraph * cgraph,+ int start_idx,+ std::initializer_list<enum ggml_op> ops,+ std::initializer_list<int> outputs = {}) {+ return ggml_can_fuse_subgraph(cgraph, start_idx, ops.size(), ops.begin(), outputs.begin(), outputs.size());+}++// Return true if the edges in the graph match expectations.+inline bool ggml_check_edges(const struct ggml_cgraph * cgraph,+ int start_idx,+ std::initializer_list<std::array<int, 3>> edges) {+ for (const auto & edge : edges) {+ int dst_node = edge[0];+ int src_idx = edge[1];+ int src_node = edge[2];+ if (cgraph->nodes[start_idx + dst_node]->src[src_idx] != cgraph->nodes[start_idx + src_node]) {+ return false;+ }+ }+ return true;+}++// expose GGUF internals for test code+GGML_API size_t gguf_type_size(enum gguf_type type);+GGML_API void gguf_write_to_buf(const struct gguf_context * ctx, std::vector<int8_t> & buf, bool only_meta);+#endif // __cplusplusdiff --git a/python/freetoken/kernel/csrc/gguf/dequantize.cuh b/python/freetoken/kernel/csrc/gguf/dequantize.cuhdeleted file mode 100644index 57cb1a27..00000000--- a/python/freetoken/kernel/csrc/gguf/dequantize.cuh+++ /dev/null@@ -1,583 +0,0 @@-// copied from-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/dequantize.cuh-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu-// Dequant functions-static __device__ __forceinline__ void dequantize_q4_0(const void* vx, const int ib, const int iqs, dfloat2& v) {- const block_q4_0* x = (const block_q4_0*)vx;-- const dfloat d = x[ib].d;-- const int vui = x[ib].qs[iqs];-- v.x = __int2half_rn(vui & 0xF);- v.y = __int2half_rn(vui >> 4);-- v = __hsub2(v, __floats2half2_rn(8.0f, 8.0f));- v = __hmul2(v, {d, d});-}--static __device__ __forceinline__ void dequantize_q4_1(const void* vx, const int ib, const int iqs, dfloat2& v) {- const block_q4_1* x = (const block_q4_1*)vx;-- const dfloat d = __low2half(x[ib].dm);- const dfloat m = __high2half(x[ib].dm);-- const int vui = x[ib].qs[iqs];-- v.x = __int2half_rn(vui & 0xF);- v.y = __int2half_rn(vui >> 4);-- v = __hmul2(v, {d, d});- v = __hadd2(v, {m, m});-}--static __device__ __forceinline__ void dequantize_q5_0(const void* vx, const int ib, const int iqs, dfloat2& v) {- const block_q5_0* x = (const block_q5_0*)vx;-- const dfloat d = x[ib].d;-- uint32_t qh;- memcpy(&qh, x[ib].qh, sizeof(qh));-- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;- const int xh_1 = ((qh >> (iqs + 12))) & 0x10;-- v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);- v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);-- v = __hsub2(v, __floats2half2_rn(16.0f, 16.0f));- v = __hmul2(v, {d, d});-}--static __device__ __forceinline__ void dequantize_q5_1(const void* vx, const int ib, const int iqs, dfloat2& v) {- const block_q5_1* x = (const block_q5_1*)vx;-- const dfloat d = __low2half(x[ib].dm);- const dfloat m = __high2half(x[ib].dm);-- uint32_t qh;- memcpy(&qh, x[ib].qh, sizeof(qh));-- const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;- const int xh_1 = ((qh >> (iqs + 12))) & 0x10;-- v.x = __int2half_rn((x[ib].qs[iqs] & 0xf) | xh_0);- v.y = __int2half_rn((x[ib].qs[iqs] >> 4) | xh_1);-- v = __hmul2(v, {d, d});- v = __hadd2(v, {m, m});-}--static __device__ __forceinline__ void dequantize_q8_0(const void* vx, const int ib, const int iqs, dfloat2& v) {- const block_q8_0* x = (const block_q8_0*)vx;-- const dfloat d = x[ib].d;-- v.x = __int2half_rn(x[ib].qs[iqs + 0]);- v.y = __int2half_rn(x[ib].qs[iqs + 1]);-- v = __hmul2(v, {d, d});-}--template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static __global__ void dequantize_block(const void* __restrict__ vx, dst_t* __restrict__ y, const int k) {- const int i = 2 * (blockDim.x * blockIdx.x + threadIdx.x);-- if (i >= k) {- return;- }-- const int ib = i / qk; // block index- const int iqs = (i % qk) / qr; // quant index- const int iybs = i - i % qk; // y block start index- const int y_offset = qr == 1 ? 1 : qk / 2;-- // dequantize- dfloat2 v;- dequantize_kernel(vx, ib, iqs, v);-- y[iybs + iqs + 0] = convert_from_half<dst_t>(v.x);- y[iybs + iqs + y_offset] = convert_from_half<dst_t>(v.y);-}--template <typename dst_t>-static __global__ void dequantize_block_q2_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_q2_K* x = (const block_q2_K*)vx;-- const auto tid = threadIdx.x;- const int n = tid / 32;- const int l = tid - 32 * n;- const int is = 8 * n + l / 16;-- const uint8_t q = x[i].qs[32 * n + l];- dst_t* y = yy + i * QK_K + 128 * n;-- half dall = __low2half(x[i].dm);- half dmin = __high2half(x[i].dm);- y[l + 0] = convert_from_half<dst_t>(__hsub(- __hmul(dall, __int2half_rn((x[i].scales[is + 0] & 0xF) * ((q >> 0) & 3))),- __hmul(dmin, __int2half_rn(x[i].scales[is + 0] >> 4))));- y[l + 32] = convert_from_half<dst_t>(__hsub(- __hmul(dall, __int2half_rn((x[i].scales[is + 2] & 0xF) * ((q >> 2) & 3))),- __hmul(dmin, __int2half_rn(x[i].scales[is + 2] >> 4))));- y[l + 64] = convert_from_half<dst_t>(__hsub(- __hmul(dall, __int2half_rn((x[i].scales[is + 4] & 0xF) * ((q >> 4) & 3))),- __hmul(dmin, __int2half_rn(x[i].scales[is + 4] >> 4))));- y[l + 96] = convert_from_half<dst_t>(__hsub(- __hmul(dall, __int2half_rn((x[i].scales[is + 6] & 0xF) * ((q >> 6) & 3))),- __hmul(dmin, __int2half_rn(x[i].scales[is + 6] >> 4))));-}--template <typename dst_t>-static __global__ void dequantize_block_q3_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_q3_K* x = (const block_q3_K*)vx;-- const auto r = threadIdx.x / 4;- const int tid = r / 2;- const int is0 = r % 2;- const int l0 = 16 * is0 + 4 * (threadIdx.x % 4);- const int n = tid / 4;- const int j = tid - 4 * n;-- uint8_t m = 1 << (4 * n + j);- int is = 8 * n + 2 * j + is0;- int shift = 2 * j;-- int8_t us = is < 4 ? (x[i].scales[is - 0] & 0xF) | (((x[i].scales[is + 8] >> 0) & 3) << 4)- : is < 8 ? (x[i].scales[is - 0] & 0xF) | (((x[i].scales[is + 4] >> 2) & 3) << 4)- : is < 12 ? (x[i].scales[is - 8] >> 4) | (((x[i].scales[is + 0] >> 4) & 3) << 4)- : (x[i].scales[is - 8] >> 4) | (((x[i].scales[is - 4] >> 6) & 3) << 4);- half d_all = x[i].d;- half dl = __hmul(d_all, __int2half_rn(us - 32));-- dst_t* y = yy + i * QK_K + 128 * n + 32 * j;- const uint8_t* q = x[i].qs + 32 * n;- const uint8_t* hm = x[i].hmask;-- for (int l = l0; l < l0 + 4; ++l) {- y[l] = convert_from_half<dst_t>(__hmul(dl, __int2half_rn((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4))));- }-}--static inline __device__ void get_scale_min_k4(int j, const uint8_t* q, uint8_t& d, uint8_t& m) {- if (j < 4) {- d = q[j] & 63;- m = q[j + 4] & 63;- } else {- d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);- m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);- }-}--template <typename dst_t>-static __global__ void dequantize_block_q4_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const block_q4_K* x = (const block_q4_K*)vx;-- const auto i = blockIdx.x;-- // assume 32 threads- const auto tid = threadIdx.x;- const int il = tid / 8;- const int ir = tid % 8;- const int is = 2 * il;- const int n = 4;-- dst_t* y = yy + i * QK_K + 64 * il + n * ir;-- const half dall = __low2half(x[i].dm);- const half dmin = __high2half(x[i].dm);-- const uint8_t* q = x[i].qs + 32 * il + n * ir;-- uint8_t sc, m;- get_scale_min_k4(is + 0, x[i].scales, sc, m);- const half d1 = __hmul(dall, __int2half_rn(sc));- const half m1 = __hmul(dmin, __int2half_rn(m));- get_scale_min_k4(is + 1, x[i].scales, sc, m);- const half d2 = __hmul(dall, __int2half_rn(sc));- const half m2 = __hmul(dmin, __int2half_rn(m));- for (int l = 0; l < n; ++l) {- y[l + 0] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn(q[l] & 0xF)), m1));- y[l + 32] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn(q[l] >> 4)), m2));- }-}--template <typename dst_t>-static __global__ void dequantize_block_q5_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const block_q5_K* x = (const block_q5_K*)vx;-- const auto i = blockIdx.x;-- // assume 64 threads - this is very slightly better than the one below- const auto tid = threadIdx.x;- const int il = tid / 16; // il is in 0...3- const int ir = tid % 16; // ir is in 0...15- const int is = 2 * il; // is is in 0...6-- dst_t* y = yy + i * QK_K + 64 * il + 2 * ir;-- const half dall = __low2half(x[i].dm);- const half dmin = __high2half(x[i].dm);-- const uint8_t* ql = x[i].qs + 32 * il + 2 * ir;- const uint8_t* qh = x[i].qh + 2 * ir;-- uint8_t sc, m;- get_scale_min_k4(is + 0, x[i].scales, sc, m);- const half d1 = __hmul(dall, __int2half_rn(sc));- const half m1 = __hmul(dmin, __int2half_rn(m));- get_scale_min_k4(is + 1, x[i].scales, sc, m);- const half d2 = __hmul(dall, __int2half_rn(sc));- const half m2 = __hmul(dmin, __int2half_rn(m));-- uint8_t hm = 1 << (2 * il);- y[0] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn((ql[0] & 0xF) + (qh[0] & hm ? 16 : 0))), m1));- y[1] = convert_from_half<dst_t>(__hsub(__hmul(d1, __int2half_rn((ql[1] & 0xF) + (qh[1] & hm ? 16 : 0))), m1));- hm <<= 1;- y[32] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn((ql[0] >> 4) + (qh[0] & hm ? 16 : 0))), m2));- y[33] = convert_from_half<dst_t>(__hsub(__hmul(d2, __int2half_rn((ql[1] >> 4) + (qh[1] & hm ? 16 : 0))), m2));-}--template <typename dst_t>-static __global__ void dequantize_block_q6_K(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const block_q6_K* x = (const block_q6_K*)vx;-- const auto i = blockIdx.x;-- // assume 64 threads - this is very slightly better than the one below- const auto tid = threadIdx.x;- const int ip = tid / 32; // ip is 0 or 1- const int il = tid - 32 * ip; // 0...32- const int is = 8 * ip + il / 16;-- dst_t* y = yy + i * QK_K + 128 * ip + il;-- const half d = x[i].d;-- const uint8_t* ql = x[i].ql + 64 * ip + il;- const uint8_t qh = x[i].qh[32 * ip + il];- const int8_t* sc = x[i].scales + is;-- y[0] = convert_from_half<dst_t>(- __hmul(d, __int2half_rn(sc[0] * ((int8_t)((ql[0] & 0xF) | (((qh >> 0) & 3) << 4)) - 32))));- y[32] = convert_from_half<dst_t>(- __hmul(d, __int2half_rn(sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32))));- y[64] = convert_from_half<dst_t>(- __hmul(d, __int2half_rn(sc[4] * ((int8_t)((ql[0] >> 4) | (((qh >> 4) & 3) << 4)) - 32))));- y[96] = convert_from_half<dst_t>(- __hmul(d, __int2half_rn(sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32))));-}--template <typename dst_t>-static __global__ void dequantize_block_iq2_xxs(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_iq2_xxs* x = (const block_iq2_xxs*)vx;-- const auto tid = threadIdx.x;- const int il = tid / 8; // 0...3- const int ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;- const uint16_t* q2 = x[i].qs + 4 * ib;- const uint8_t* aux8 = (const uint8_t*)q2;- const uint8_t* grid = (const uint8_t*)(iq2xxs_grid + aux8[il]);- const uint32_t aux32 = q2[2] | (q2[3] << 16);- const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.25f;- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7 * il) & 127];- for (int j = 0; j < 8; ++j)- y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);-}--template <typename dst_t>-static __global__ void dequantize_block_iq2_xs(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_iq2_xs* x = (const block_iq2_xs*)vx;-- const auto tid = threadIdx.x;- const int il = tid / 8; // 0...3- const int ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;- const uint16_t* q2 = x[i].qs + 4 * ib;- const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[il] & 511));- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4 * (il / 2)) & 0xf)) * 0.25f;- const uint8_t signs = ksigns_iq2xs[q2[il] >> 9];- for (int j = 0; j < 8; ++j)- y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);-}--template <typename dst_t>-static __global__ void dequantize_block_iq2_s(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_iq2_s* x = (const block_iq2_s*)vx;-- const auto tid = threadIdx.x;- const int il = tid / 8; // 0...3- const int ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;- const uint8_t* grid = (const uint8_t*)(iq2s_grid + (x[i].qs[4 * ib + il] | ((x[i].qh[ib] << (8 - 2 * il)) & 0x300)));- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib] >> 4 * (il / 2)) & 0xf)) * 0.25f;- const uint8_t signs = x[i].qs[QK_K / 8 + 4 * ib + il];- for (int j = 0; j < 8; ++j)- y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f);-}--template <typename dst_t>-static __global__ void dequantize_block_iq3_xxs(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_iq3_xxs* x = (const block_iq3_xxs*)vx;-- const auto tid = threadIdx.x;- const int il = tid / 8; // 0...3- const int ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;- const uint8_t* q3 = x[i].qs + 8 * ib;- const uint16_t* gas = (const uint16_t*)(x[i].qs + QK_K / 4) + 2 * ib;- const uint8_t* grid1 = (const uint8_t*)(iq3xxs_grid + q3[2 * il + 0]);- const uint8_t* grid2 = (const uint8_t*)(iq3xxs_grid + q3[2 * il + 1]);- const uint32_t aux32 = gas[0] | (gas[1] << 16);- const float d = __half2float(x[i].d) * (0.5f + (aux32 >> 28)) * 0.5f;- const uint8_t signs = ksigns_iq2xs[(aux32 >> 7 * il) & 127];- for (int j = 0; j < 4; ++j) {- y[j + 0] = d * grid1[j] * (signs & kmask_iq2xs[j + 0] ? -1.f : 1.f);- y[j + 4] = d * grid2[j] * (signs & kmask_iq2xs[j + 4] ? -1.f : 1.f);- }-}--template <typename dst_t>-static __global__ void dequantize_block_iq3_s(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_iq3_s* x = (const block_iq3_s*)vx;-- const auto tid = threadIdx.x;- const int il = tid / 8; // 0...3- const int ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;- const uint8_t* qs = x[i].qs + 8 * ib;- const uint8_t* grid1 = (const uint8_t*)(iq3xs_grid + (qs[2 * il + 0] | ((x[i].qh[ib] << (8 - 2 * il)) & 256)));- const uint8_t* grid2 = (const uint8_t*)(iq3xs_grid + (qs[2 * il + 1] | ((x[i].qh[ib] << (7 - 2 * il)) & 256)));- const float d = __half2float(x[i].d) * (0.5f + ((x[i].scales[ib / 2] >> 4 * (ib % 2)) & 0xf)) * 0.5f;- const uint8_t signs = x[i].signs[4 * ib + il];- for (int j = 0; j < 4; ++j) {- y[j + 0] = d * grid1[j] * (signs & kmask_iq2xs[j + 0] ? -1.f : 1.f);- y[j + 4] = d * grid2[j] * (signs & kmask_iq2xs[j + 4] ? -1.f : 1.f);- }-}--template <typename dst_t>-static __global__ void dequantize_block_iq1_s(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const int64_t i = blockIdx.x;- const block_iq1_s* x = (const block_iq1_s*)vx;-- const int64_t tid = threadIdx.x;- const int64_t il = tid / 8; // 0...3- const int64_t ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;- const float delta = x[i].qh[ib] & 0x8000 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA;- const float d = __half2float(x[i].d) * (2 * ((x[i].qh[ib] >> 12) & 7) + 1);- uint32_t grid32[2];- const int8_t* q = (const int8_t*)grid32;- grid32[0] = iq1s_grid_gpu[x[i].qs[4 * ib + il] | (((x[i].qh[ib] >> 3 * il) & 7) << 8)];- grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;- grid32[0] &= 0x0f0f0f0f;- for (int j = 0; j < 8; ++j) {- y[j] = d * (q[j] + delta);- }-}--template <typename dst_t>-static __global__ void dequantize_block_iq1_m(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const int64_t i = blockIdx.x;- const block_iq1_m* x = (const block_iq1_m*)vx;-- const int64_t tid = threadIdx.x;- const int64_t il = tid / 8; // 0...3- const int64_t ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 8 * il;- const uint16_t* sc = (const uint16_t*)x[i].scales;- iq1m_scale_t scale;- scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000);- const int64_t ib16 = 2 * ib + il / 2; // sc[ib16/4] >> 3*(ib16%4) -> sc[ib/2] >> 3*((2*ib+il/2)%4);- const float d = __half2float(scale.f16) * (2 * ((sc[ib16 / 4] >> 3 * (ib16 % 4)) & 0x7) + 1);- const float delta = x[i].qh[2 * ib + il / 2] & (0x08 << 4 * (il % 2)) ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA;- uint32_t grid32[2];- const int8_t* q = (const int8_t*)grid32;- grid32[0] = iq1s_grid_gpu[x[i].qs[4 * ib + il] | (((x[i].qh[2 * ib + il / 2] >> 4 * (il % 2)) & 7) << 8)];- grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f;- grid32[0] &= 0x0f0f0f0f;- for (int j = 0; j < 8; ++j) {- y[j] = d * (q[j] + delta);- }-}--template <typename dst_t>-static __global__ void dequantize_block_iq4_nl(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_iq4_nl* x = (const block_iq4_nl*)vx + i * (QK_K / QK4_NL);-- const auto tid = threadIdx.x;- const int il = tid / 8; // 0...3- const int ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 4 * il;- const uint8_t* q4 = x[ib].qs + 4 * il;- const float d = __half2float(x[ib].d);- for (int j = 0; j < 4; ++j) {- y[j + 0] = d * kvalues_iq4nl[q4[j] & 0xf];- y[j + 16] = d * kvalues_iq4nl[q4[j] >> 4];- }-}--template <typename dst_t>-static __global__ void dequantize_block_iq4_xs(const void* __restrict__ vx, dst_t* __restrict__ yy) {- const auto i = blockIdx.x;- const block_iq4_xs* x = (const block_iq4_xs*)vx;-- const auto tid = threadIdx.x;- const int il = tid / 8; // 0...3- const int ib = tid % 8; // 0...7- dst_t* y = yy + i * QK_K + 32 * ib + 4 * il;- const uint8_t* q4 = x[i].qs + 16 * ib + 4 * il;- const float d = __half2float(x[i].d) *- ((((x[i].scales_l[ib / 2] >> 4 * (ib % 2)) & 0xf) | (((x[i].scales_h >> 2 * ib) & 3) << 4)) - 32);- for (int j = 0; j < 4; ++j) {- y[j + 0] = d * kvalues_iq4nl[q4[j] & 0xf];- y[j + 16] = d * kvalues_iq4nl[q4[j] >> 4];- }-}--template <int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>-static void-dequantize_block_cuda(const void* __restrict__ vx, dst_t* __restrict__ y, const int k, cudaStream_t stream) {- const int num_blocks = (k + 2 * CUDA_DEQUANTIZE_BLOCK_SIZE - 1) / (2 * CUDA_DEQUANTIZE_BLOCK_SIZE);- dequantize_block<qk, qr, dequantize_kernel><<<num_blocks, CUDA_DEQUANTIZE_BLOCK_SIZE, 0, stream>>>(vx, y, k);-}--template <typename dst_t>-static void dequantize_row_q2_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_q2_K<<<nb, 64, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_q3_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_q3_K<<<nb, 64, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_q4_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_q4_K<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_q5_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_q5_K<<<nb, 64, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_q6_K_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_q6_K<<<nb, 64, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq2_xxs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_xxs<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq2_xs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_xs<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq2_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq2_s<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq3_xxs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq3_xxs<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq3_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq3_s<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq1_s_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq1_s<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq1_m_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = k / QK_K;- dequantize_block_iq1_m<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq4_nl_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = (k + QK_K - 1) / QK_K;- dequantize_block_iq4_nl<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static void dequantize_row_iq4_xs_cuda(const void* vx, dst_t* y, const int k, cudaStream_t stream) {- const int nb = (k + QK_K - 1) / QK_K;- dequantize_block_iq4_xs<<<nb, 32, 0, stream>>>(vx, y);-}--template <typename dst_t>-static to_cuda_ggml_t<dst_t> ggml_get_to_cuda(int64_t type) {- switch (type) {- case 2:- return dequantize_block_cuda<QK4_0, QR4_0, dequantize_q4_0>;- case 3:- return dequantize_block_cuda<QK4_1, QR4_1, dequantize_q4_1>;- case 6:- return dequantize_block_cuda<QK5_0, QR5_0, dequantize_q5_0>;- case 7:- return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;- case 8:- return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;- case 10:- return dequantize_row_q2_K_cuda;- case 11:- return dequantize_row_q3_K_cuda;- case 12:- return dequantize_row_q4_K_cuda;- case 13:- return dequantize_row_q5_K_cuda;- case 14:- return dequantize_row_q6_K_cuda;- case 16:- return dequantize_row_iq2_xxs_cuda;- case 17:- return dequantize_row_iq2_xs_cuda;- case 18:- return dequantize_row_iq3_xxs_cuda;- case 19:- return dequantize_row_iq1_s_cuda;- case 20:- return dequantize_row_iq4_nl_cuda;- case 21:- return dequantize_row_iq3_s_cuda;- case 22:- return dequantize_row_iq2_s_cuda;- case 23:- return dequantize_row_iq4_xs_cuda;- case 29:- return dequantize_row_iq1_m_cuda;- default:- return nullptr;- }-}diff --git a/python/freetoken/kernel/csrc/gguf/dispatch.h b/python/freetoken/kernel/csrc/gguf/dispatch.hdeleted file mode 100644index f42a2163..00000000--- a/python/freetoken/kernel/csrc/gguf/dispatch.h+++ /dev/null@@ -1,28 +0,0 @@-// Minimal AT_DISPATCH helper for the vendored GGUF kernels (borrowed from-// sgl-kernel csrc/quantization/gguf, which are ports of llama.cpp). The donor-// pulls these macros from its large include/utils.h; we only need the float-// dispatch, so vendor just that to keep the JIT compile self-contained.-#pragma once--#include <ATen/Dispatch.h>--#ifndef WARP_SIZE-#define WARP_SIZE 32-#endif--// Warp-shuffle wrappers the donor pulls from sgl-kernel's utils.h (CUDA variants).-#ifndef SGLANG_SHFL_XOR_SYNC-#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask))-#endif-#ifndef SGLANG_SHFL_XOR_SYNC_WIDTH-#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) \- __shfl_xor_sync((mask), (var), (lane_mask), (width))-#endif--#define DISPATCH_CASE_FLOAT_TYPES(...) \- AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \- AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \- AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)--#define DISPATCH_FLOAT_TYPES(TYPE, NAME, ...) \- AT_DISPATCH_SWITCH(TYPE, NAME, DISPATCH_CASE_FLOAT_TYPES(__VA_ARGS__))diff --git a/python/freetoken/kernel/csrc/gguf/ggml-common.h b/python/freetoken/kernel/csrc/gguf/ggml-common.hdeleted file mode 100644index 88c21a4a..00000000--- a/python/freetoken/kernel/csrc/gguf/ggml-common.h+++ /dev/null@@ -1,1029 +0,0 @@-// adapted from-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/ggml-common.h-// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h-#define QK_K 256-#define K_QUANTS_PER_ITERATION 2-#define WARP_SIZE_GGUF 32-#define K_SCALE_SIZE 12-#define CUDA_DEQUANTIZE_BLOCK_SIZE 256-#define CUDA_QUANTIZE_BLOCK_SIZE 256-#define GGML_CUDA_DMMV_X 32-#define GGML_CUDA_MMV_Y 1--// Data Structures-// QK = number of values after dequantization-// QR = QK / number of values before dequantization-// QI = number of 32 bit integers before dequantization--#define QK4_0 32-#define QR4_0 2-#define QI4_0 (QK4_0 / (4 * QR4_0))-typedef struct {- half d; // delta- uint8_t qs[QK4_0 / 2]; // nibbles / quants-} block_q4_0;--#define QK4_1 32-#define QR4_1 2-#define QI4_1 (QK4_1 / (4 * QR4_1))-typedef struct {- half2 dm; // dm.x = delta, dm.y = min- uint8_t qs[QK4_1 / 2]; // nibbles / quants-} block_q4_1;--#define QK5_0 32-#define QR5_0 2-#define QI5_0 (QK5_0 / (4 * QR5_0))-typedef struct {- half d; // delta- uint8_t qh[4]; // 5-th bit of quants- uint8_t qs[QK5_0 / 2]; // nibbles / quants-} block_q5_0;--#define QK5_1 32-#define QR5_1 2-#define QI5_1 (QK5_1 / (4 * QR5_1))-typedef struct {- half2 dm; // dm.x = delta, dm.y = min- uint8_t qh[4]; // 5-th bit of quants- uint8_t qs[QK5_1 / 2]; // nibbles / quants-} block_q5_1;--#define QK8_0 32-#define QR8_0 1-#define QI8_0 (QK8_0 / (4 * QR8_0))-typedef struct {- half d; // delta- int8_t qs[QK8_0]; // quants-} block_q8_0;--#define QK8_1 32-#define QR8_1 1-#define QI8_1 (QK8_1 / (4 * QR8_1))-typedef struct {- half2 ds; // ds.x = delta, ds.y = sum- int8_t qs[QK8_0]; // quants-} block_q8_1;--#define QR2_K 4-#define QI2_K (QK_K / (4 * QR2_K))-typedef struct {- uint8_t scales[QK_K / 16]; // scales and mins, quantized with 4 bits- uint8_t qs[QK_K / 4]; // quants- half2 dm; // super-block scale for quantized scales/mins-} block_q2_K;--#define QR3_K 4-#define QI3_K (QK_K / (4 * QR3_K))-typedef struct {- uint8_t hmask[QK_K / 8]; // quants - high bit- uint8_t qs[QK_K / 4]; // quants - low 2 bits- uint8_t scales[K_SCALE_SIZE]; // scales, quantized with 6 bits- half d; // super-block scale-} block_q3_K;--#define QR4_K 2-#define QI4_K (QK_K / (4 * QR4_K))-typedef struct {- half2 dm; // super-block scale for quantized scales/mins- uint8_t scales[3 * QK_K / 64]; // scales, quantized with 6 bits- uint8_t qs[QK_K / 2]; // 4--bit quants-} block_q4_K;--#define QR5_K 2-#define QI5_K (QK_K / (4 * QR5_K))-typedef struct {- half2 dm; // super-block scale for quantized scales/mins- uint8_t scales[K_SCALE_SIZE]; // scales and mins, quantized with 6 bits- uint8_t qh[QK_K / 8]; // quants, high bit- uint8_t qs[QK_K / 2]; // quants, low 4 bits-} block_q5_K;--#define QR6_K 2-#define QI6_K (QK_K / (4 * QR6_K))-typedef struct {- uint8_t ql[QK_K / 2]; // quants, lower 4 bits- uint8_t qh[QK_K / 4]; // quants, upper 2 bits- int8_t scales[QK_K / 16]; // scales- half d; // delta-} block_q6_K;--#define QR2_XXS 8-#define QI2_XXS (QK_K / (4 * QR2_XXS))-typedef struct {- half d;- uint16_t qs[QK_K / 8];-} block_iq2_xxs;--#define QR2_XS 8-#define QI2_XS (QK_K / (4 * QR2_XS))-typedef struct {- half d;- uint16_t qs[QK_K / 8];- uint8_t scales[QK_K / 32];-} block_iq2_xs;--#define QR2_S 8-#define QI2_S (QK_K / (4 * QR2_S))-typedef struct {- half d;- uint8_t qs[QK_K / 4];- uint8_t qh[QK_K / 32];- uint8_t scales[QK_K / 32];-} block_iq2_s;--#define QR3_XXS 8-#define QI3_XXS (QK_K / (4 * QR3_XXS))-typedef struct {- half d;- uint8_t qs[3 * (QK_K / 8)];-} block_iq3_xxs;--#define QR3_XS 8-#define QI3_XS (QK_K / (4 * QR3_XS))-#define IQ3S_N_SCALE QK_K / 64-typedef struct {- half d;- uint8_t qs[QK_K / 4];- uint8_t qh[QK_K / 32];- uint8_t signs[QK_K / 8];- uint8_t scales[IQ3S_N_SCALE];-} block_iq3_s;--// 1.5625 bpw-#define QR1_S 8-#define QI1_S (QK_K / (4 * QR1_S))-typedef struct {- half d;- uint8_t qs[QK_K / 8];- uint16_t qh[QK_K / 32];-} block_iq1_s;--// 1.75 bpw-#define QR1_M 8-#define QI1_M (QK_K / (4 * QR1_M))-typedef struct {- uint8_t qs[QK_K / 8]; // grid index, low 8 bits- uint8_t qh[QK_K / 16]; // grid index, high 3 bits + grid shift bit (for two groups of 8)- uint8_t scales[QK_K / 32]; // 3-bit block scales (4-bit if QK_K == 64)-} block_iq1_m;--// Used by IQ1_M quants-typedef union {- half f16;- uint16_t u16;-} iq1m_scale_t;--#define QK4_NL 32-#define QR4_NL 2-#define QI4_NL (QK4_NL / (4 * QR4_NL))-typedef struct {- half d;- uint8_t qs[QK4_NL / 2];-} block_iq4_nl;--#define QR4_XS 8-#define QI4_XS (QK_K / (4 * QR4_XS))-typedef struct {- half d;- uint16_t scales_h;- uint8_t scales_l[QK_K / 64];- uint8_t qs[QK_K / 2];-} block_iq4_xs;--static const __device__ uint64_t iq2xxs_grid[256] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b,- 0x0808080808190819, 0x0808080808191908, 0x08080808082b0808, 0x08080808082b082b, 0x08080808082b2b08,- 0x08080808082b2b2b, 0x0808080819080819, 0x0808080819081908, 0x0808080819190808, 0x0808080819192b08,- 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b, 0x080808082b082b2b,- 0x080808082b2b082b, 0x0808081908080819, 0x0808081908081908, 0x0808081908190808, 0x0808081908191919,- 0x0808081919080808, 0x080808192b081908, 0x080808192b192b08, 0x0808082b08080808, 0x0808082b0808082b,- 0x0808082b082b082b, 0x0808082b2b08082b, 0x0808190808080819, 0x0808190808081908, 0x0808190808190808,- 0x08081908082b0819, 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819082b08,- 0x08081908192b0808, 0x080819082b080819, 0x080819082b081908, 0x080819082b190808, 0x080819082b2b1908,- 0x0808191908080808, 0x080819190808082b, 0x0808191908082b08, 0x08081919082b0808, 0x080819191908192b,- 0x08081919192b2b19, 0x080819192b080808, 0x080819192b190819, 0x0808192b08082b19, 0x0808192b08190808,- 0x0808192b19080808, 0x0808192b2b081908, 0x0808192b2b2b1908, 0x08082b0808080808, 0x08082b0808081919,- 0x08082b0808082b08, 0x08082b0808191908, 0x08082b08082b2b08, 0x08082b0819080819, 0x08082b0819081908,- 0x08082b0819190808, 0x08082b081919082b, 0x08082b082b082b08, 0x08082b1908081908, 0x08082b1919080808,- 0x08082b2b0808082b, 0x08082b2b08191908, 0x0819080808080819, 0x0819080808081908, 0x0819080808190808,- 0x08190808082b0819, 0x0819080819080808, 0x08190808192b0808, 0x081908082b081908, 0x081908082b190808,- 0x081908082b191919, 0x0819081908080808, 0x0819081908082b08, 0x08190819082b0808, 0x0819081919190808,- 0x0819081919192b2b, 0x081908192b080808, 0x0819082b082b1908, 0x0819082b19081919, 0x0819190808080808,- 0x0819190808082b08, 0x08191908082b0808, 0x08191908082b1919, 0x0819190819082b19, 0x081919082b080808,- 0x0819191908192b08, 0x08191919192b082b, 0x0819192b08080808, 0x0819192b0819192b, 0x08192b0808080819,- 0x08192b0808081908, 0x08192b0808190808, 0x08192b0819080808, 0x08192b082b080819, 0x08192b1908080808,- 0x08192b1908081919, 0x08192b192b2b0808, 0x08192b2b19190819, 0x082b080808080808, 0x082b08080808082b,- 0x082b080808082b2b, 0x082b080819081908, 0x082b0808192b0819, 0x082b08082b080808, 0x082b08082b08082b,- 0x082b0819082b2b19, 0x082b081919082b08, 0x082b082b08080808, 0x082b082b0808082b, 0x082b190808080819,- 0x082b190808081908, 0x082b190808190808, 0x082b190819080808, 0x082b19081919192b, 0x082b191908080808,- 0x082b191919080819, 0x082b1919192b1908, 0x082b192b2b190808, 0x082b2b0808082b08, 0x082b2b08082b0808,- 0x082b2b082b191908, 0x082b2b2b19081908, 0x1908080808080819, 0x1908080808081908, 0x1908080808190808,- 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808, 0x1908080819082b08,- 0x190808081919192b, 0x19080808192b0808, 0x190808082b080819, 0x190808082b081908, 0x190808082b190808,- 0x1908081908080808, 0x19080819082b0808, 0x19080819192b0819, 0x190808192b080808, 0x190808192b081919,- 0x1908082b08080819, 0x1908082b08190808, 0x1908082b19082b08, 0x1908082b1919192b, 0x1908082b192b2b08,- 0x1908190808080808, 0x1908190808082b08, 0x19081908082b0808, 0x190819082b080808, 0x190819082b192b19,- 0x190819190819082b, 0x19081919082b1908, 0x1908192b08080808, 0x19082b0808080819, 0x19082b0808081908,- 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b1908080808, 0x19082b1919192b08,- 0x19082b19192b0819, 0x19082b192b08082b, 0x19082b2b19081919, 0x19082b2b2b190808, 0x1919080808080808,- 0x1919080808082b08, 0x1919080808190819, 0x1919080808192b19, 0x19190808082b0808, 0x191908082b080808,- 0x191908082b082b08, 0x1919081908081908, 0x191908191908082b, 0x191908192b2b1908, 0x1919082b2b190819,- 0x191919082b190808, 0x191919082b19082b, 0x1919191908082b2b, 0x1919192b08080819, 0x1919192b19191908,- 0x19192b0808080808, 0x19192b0808190819, 0x19192b0808192b19, 0x19192b08192b1908, 0x19192b1919080808,- 0x19192b2b08082b08, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808, 0x192b0808192b2b08,- 0x192b081908080808, 0x192b081919191919, 0x192b082b08192b08, 0x192b082b192b0808, 0x192b190808080808,- 0x192b190808081919, 0x192b191908190808, 0x192b19190819082b, 0x192b19192b081908, 0x192b2b081908082b,- 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808082b2b, 0x2b08080819080819, 0x2b0808082b08082b,- 0x2b08081908081908, 0x2b08081908192b08, 0x2b08081919080808, 0x2b08082b08190819, 0x2b08190808080819,- 0x2b08190808081908, 0x2b08190808190808, 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808,- 0x2b08191908080808, 0x2b0819191908192b, 0x2b0819192b191908, 0x2b08192b08082b19, 0x2b08192b19080808,- 0x2b08192b192b0808, 0x2b082b080808082b, 0x2b082b1908081908, 0x2b082b2b08190819, 0x2b19080808081908,- 0x2b19080808190808, 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908082b2b0819, 0x2b1908190819192b,- 0x2b1908192b080808, 0x2b19082b19081919, 0x2b19190808080808, 0x2b191908082b082b, 0x2b19190819081908,- 0x2b19191919190819, 0x2b192b082b080819, 0x2b192b19082b0808, 0x2b2b08080808082b, 0x2b2b080819190808,- 0x2b2b08082b081919, 0x2b2b081908082b19, 0x2b2b082b08080808, 0x2b2b190808192b08, 0x2b2b2b0819190808,- 0x2b2b2b1908081908,-};--static const __device__ uint64_t iq2xs_grid[512] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b,- 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, 0x0808080808192b19, 0x08080808082b0808,- 0x08080808082b082b, 0x08080808082b1919, 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908,- 0x080808081908192b, 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x080808082b080808, 0x080808082b08082b,- 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819, 0x080808082b191908, 0x080808082b192b19,- 0x080808082b2b0808, 0x0808081908080819, 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19,- 0x0808081908190808, 0x080808190819082b, 0x0808081908191919, 0x0808081908192b08, 0x0808081908192b2b,- 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808, 0x080808191908082b, 0x0808081919081919,- 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908, 0x08080819192b0808, 0x08080819192b2b08,- 0x080808192b080819, 0x080808192b081908, 0x080808192b190808, 0x0808082b08080808, 0x0808082b0808082b,- 0x0808082b08081919, 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808,- 0x0808082b19080819, 0x0808082b19081908, 0x0808082b19190808, 0x0808082b19191919, 0x0808082b2b080808,- 0x0808082b2b082b2b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, 0x0808190808082b19,- 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819,- 0x08081908082b1908, 0x0808190819080808, 0x080819081908082b, 0x0808190819081919, 0x0808190819082b08,- 0x0808190819190819, 0x0808190819191908, 0x080819081919192b, 0x08081908192b0808, 0x080819082b080819,- 0x080819082b081908, 0x080819082b190808, 0x0808191908080808, 0x080819190808082b, 0x0808191908081919,- 0x0808191908082b08, 0x0808191908190819, 0x0808191908191908, 0x08081919082b0808, 0x0808191919080819,- 0x0808191919081908, 0x0808191919190808, 0x08081919192b0819, 0x080819192b080808, 0x0808192b08080819,- 0x0808192b08081908, 0x0808192b08190808, 0x0808192b082b192b, 0x0808192b19080808, 0x0808192b1908082b,- 0x0808192b2b081908, 0x08082b0808080808, 0x08082b080808082b, 0x08082b0808081919, 0x08082b0808082b08,- 0x08082b0808082b2b, 0x08082b0808190819, 0x08082b0808191908, 0x08082b08082b0808, 0x08082b08082b1919,- 0x08082b0819080819, 0x08082b0819081908, 0x08082b0819190808, 0x08082b0819192b08, 0x08082b082b080808,- 0x08082b082b2b0808, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908, 0x08082b1908190808,- 0x08082b1919080808, 0x08082b192b080819, 0x08082b192b082b19, 0x08082b2b08080808, 0x08082b2b082b0808,- 0x08082b2b082b2b08, 0x08082b2b2b19192b, 0x08082b2b2b2b0808, 0x0819080808080819, 0x0819080808081908,- 0x081908080808192b, 0x0819080808082b19, 0x0819080808190808, 0x081908080819082b, 0x0819080808191919,- 0x0819080808192b08, 0x08190808082b0819, 0x08190808082b1908, 0x0819080819080808, 0x081908081908082b,- 0x0819080819081919, 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x08190808192b0808,- 0x08190808192b2b2b, 0x081908082b080819, 0x081908082b081908, 0x081908082b190808, 0x0819081908080808,- 0x081908190808082b, 0x0819081908081919, 0x0819081908082b08, 0x0819081908190819, 0x0819081908191908,- 0x08190819082b0808, 0x0819081919080819, 0x0819081919081908, 0x0819081919190808, 0x081908192b080808,- 0x081908192b191908, 0x081908192b19192b, 0x0819082b08080819, 0x0819082b08081908, 0x0819082b0808192b,- 0x0819082b08190808, 0x0819082b19080808, 0x0819082b192b0808, 0x0819190808080808, 0x081919080808082b,- 0x0819190808081919, 0x0819190808082b08, 0x0819190808190819, 0x0819190808191908, 0x08191908082b0808,- 0x0819190819080819, 0x0819190819081908, 0x0819190819082b19, 0x0819190819190808, 0x08191908192b1908,- 0x081919082b080808, 0x0819191908080819, 0x0819191908081908, 0x0819191908190808, 0x0819191919080808,- 0x0819192b08080808, 0x0819192b08191908, 0x0819192b19082b19, 0x08192b0808080819, 0x08192b0808081908,- 0x08192b0808190808, 0x08192b080819082b, 0x08192b0819080808, 0x08192b0819191908, 0x08192b082b08192b,- 0x08192b1908080808, 0x08192b1908081919, 0x08192b19192b192b, 0x08192b2b19190819, 0x08192b2b2b2b2b19,- 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919, 0x082b080808082b08, 0x082b080808082b2b,- 0x082b080808190819, 0x082b080808191908, 0x082b0808082b0808, 0x082b080819080819, 0x082b080819081908,- 0x082b080819190808, 0x082b08082b080808, 0x082b08082b2b0808, 0x082b081908080819, 0x082b081908081908,- 0x082b081908190808, 0x082b081919080808, 0x082b081919082b08, 0x082b0819192b1919, 0x082b082b08080808,- 0x082b082b082b082b, 0x082b082b2b080808, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,- 0x082b190808190808, 0x082b1908082b2b19, 0x082b190819080808, 0x082b191908080808, 0x082b191919080819,- 0x082b19191919082b, 0x082b19192b192b19, 0x082b192b08080819, 0x082b192b08192b2b, 0x082b192b2b2b192b,- 0x082b2b0808080808, 0x082b2b0808082b08, 0x082b2b0808082b2b, 0x082b2b08082b0808, 0x082b2b0819191919,- 0x082b2b082b082b08, 0x082b2b082b2b082b, 0x082b2b19192b2b08, 0x082b2b192b190808, 0x082b2b2b08082b08,- 0x082b2b2b082b0808, 0x082b2b2b2b08082b, 0x082b2b2b2b082b08, 0x082b2b2b2b082b2b, 0x1908080808080819,- 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19, 0x1908080808190808, 0x190808080819082b,- 0x1908080808191919, 0x1908080808192b08, 0x19080808082b0819, 0x19080808082b1908, 0x1908080819080808,- 0x190808081908082b, 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, 0x1908080819190819,- 0x1908080819191908, 0x19080808192b0808, 0x19080808192b1919, 0x190808082b080819, 0x190808082b081908,- 0x190808082b190808, 0x1908081908080808, 0x190808190808082b, 0x1908081908081919, 0x1908081908082b08,- 0x1908081908190819, 0x1908081908191908, 0x19080819082b0808, 0x1908081919080819, 0x1908081919081908,- 0x1908081919190808, 0x190808192b080808, 0x190808192b081919, 0x190808192b2b082b, 0x1908082b08080819,- 0x1908082b08081908, 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b082b2b19, 0x1908082b19080808,- 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808190819,- 0x1908190808191908, 0x1908190808192b19, 0x19081908082b0808, 0x1908190819080819, 0x1908190819081908,- 0x1908190819190808, 0x190819082b080808, 0x190819082b191908, 0x1908191908080819, 0x1908191908081908,- 0x1908191908190808, 0x19081919082b1908, 0x1908191919080808, 0x190819192b192b2b, 0x1908192b08080808,- 0x1908192b08082b2b, 0x1908192b19081908, 0x1908192b19190808, 0x19082b0808080819, 0x19082b0808081908,- 0x19082b0808190808, 0x19082b0819080808, 0x19082b0819081919, 0x19082b0819191908, 0x19082b08192b082b,- 0x19082b1908080808, 0x19082b1908190819, 0x19082b1919081908, 0x19082b1919190808, 0x19082b19192b2b19,- 0x19082b2b08081908, 0x1919080808080808, 0x191908080808082b, 0x1919080808081919, 0x1919080808082b08,- 0x1919080808190819, 0x1919080808191908, 0x19190808082b0808, 0x19190808082b2b08, 0x1919080819080819,- 0x1919080819081908, 0x1919080819190808, 0x191908082b080808, 0x1919081908080819, 0x1919081908081908,- 0x1919081908190808, 0x1919081908191919, 0x1919081919080808, 0x191908191908082b, 0x1919082b08080808,- 0x1919082b19081908, 0x1919082b2b2b2b2b, 0x1919190808080819, 0x1919190808081908, 0x1919190808190808,- 0x19191908082b0819, 0x1919190819080808, 0x19191908192b0808, 0x191919082b080819, 0x191919082b2b0819,- 0x1919191908080808, 0x1919191908082b08, 0x191919192b080808, 0x191919192b082b08, 0x1919192b082b0819,- 0x1919192b192b2b08, 0x1919192b2b2b0819, 0x19192b0808080808, 0x19192b0808191908, 0x19192b0819080819,- 0x19192b0819190808, 0x19192b082b192b19, 0x19192b1908192b2b, 0x19192b1919080808, 0x19192b191908082b,- 0x19192b2b2b081919, 0x192b080808080819, 0x192b080808081908, 0x192b080808190808, 0x192b080819080808,- 0x192b080819191908, 0x192b0808192b082b, 0x192b08082b08192b, 0x192b08082b2b2b19, 0x192b081908080808,- 0x192b082b082b1908, 0x192b082b19082b2b, 0x192b082b2b19082b, 0x192b190808080808, 0x192b19080819192b,- 0x192b191908190808, 0x192b191919080808, 0x192b191919081919, 0x192b19192b2b1908, 0x192b2b0808080819,- 0x192b2b08192b2b2b, 0x192b2b19082b1919, 0x192b2b2b0808192b, 0x192b2b2b19191908, 0x192b2b2b192b082b,- 0x2b08080808080808, 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819,- 0x2b08080808191908, 0x2b080808082b0808, 0x2b080808082b2b2b, 0x2b08080819080819, 0x2b08080819081908,- 0x2b08080819190808, 0x2b0808082b080808, 0x2b0808082b08082b, 0x2b0808082b2b2b08, 0x2b0808082b2b2b2b,- 0x2b08081908080819, 0x2b08081908081908, 0x2b0808190808192b, 0x2b08081908190808, 0x2b08081919080808,- 0x2b08081919190819, 0x2b08081919192b19, 0x2b08082b08080808, 0x2b08082b082b0808, 0x2b08082b2b080808,- 0x2b08082b2b08082b, 0x2b08082b2b2b0808, 0x2b08082b2b2b2b08, 0x2b08190808080819, 0x2b08190808081908,- 0x2b08190808190808, 0x2b0819080819082b, 0x2b08190808191919, 0x2b08190819080808, 0x2b081908192b0808,- 0x2b0819082b082b19, 0x2b08191908080808, 0x2b08191919081908, 0x2b0819192b2b1919, 0x2b08192b08192b08,- 0x2b08192b192b2b2b, 0x2b082b0808080808, 0x2b082b0808082b08, 0x2b082b08082b1919, 0x2b082b0819192b2b,- 0x2b082b082b080808, 0x2b082b082b08082b, 0x2b082b082b2b2b08, 0x2b082b190808192b, 0x2b082b2b082b082b,- 0x2b082b2b2b080808, 0x2b082b2b2b082b08, 0x2b082b2b2b19192b, 0x2b082b2b2b2b2b08, 0x2b19080808080819,- 0x2b19080808081908, 0x2b19080808190808, 0x2b19080819080808, 0x2b1908081919192b, 0x2b1908082b081908,- 0x2b19081908080808, 0x2b190819082b082b, 0x2b190819192b1908, 0x2b19082b1919192b, 0x2b19082b2b082b19,- 0x2b19190808080808, 0x2b19190808081919, 0x2b19190819081908, 0x2b19190819190808, 0x2b19190819192b08,- 0x2b191919082b2b19, 0x2b1919192b190808, 0x2b1919192b19082b, 0x2b19192b19080819, 0x2b192b0819190819,- 0x2b192b082b2b192b, 0x2b192b1919082b19, 0x2b192b2b08191919, 0x2b192b2b192b0808, 0x2b2b080808080808,- 0x2b2b08080808082b, 0x2b2b080808082b08, 0x2b2b080808082b2b, 0x2b2b0808082b0808, 0x2b2b0808082b2b2b,- 0x2b2b08082b2b0808, 0x2b2b081919190819, 0x2b2b081919192b19, 0x2b2b08192b2b192b, 0x2b2b082b08080808,- 0x2b2b082b0808082b, 0x2b2b082b08082b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b080808, 0x2b2b082b2b2b0808,- 0x2b2b190819080808, 0x2b2b19082b191919, 0x2b2b192b192b1919, 0x2b2b192b2b192b08, 0x2b2b2b0808082b2b,- 0x2b2b2b08082b0808, 0x2b2b2b08082b082b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b0808, 0x2b2b2b082b2b2b08,- 0x2b2b2b1908081908, 0x2b2b2b192b081908, 0x2b2b2b192b08192b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b082b2b2b,- 0x2b2b2b2b2b190819, 0x2b2b2b2b2b2b2b2b,-};--static const __device__ uint64_t iq2s_grid[1024] = {- 0x0808080808080808, 0x080808080808082b, 0x0808080808081919, 0x0808080808082b08, 0x0808080808082b2b,- 0x0808080808190819, 0x0808080808191908, 0x080808080819192b, 0x0808080808192b19, 0x08080808082b0808,- 0x08080808082b082b, 0x08080808082b1919, 0x08080808082b2b08, 0x0808080819080819, 0x0808080819081908,- 0x080808081908192b, 0x0808080819082b19, 0x0808080819190808, 0x080808081919082b, 0x0808080819191919,- 0x0808080819192b08, 0x08080808192b0819, 0x08080808192b1908, 0x08080808192b192b, 0x08080808192b2b19,- 0x080808082b080808, 0x080808082b08082b, 0x080808082b081919, 0x080808082b082b08, 0x080808082b190819,- 0x080808082b191908, 0x080808082b2b0808, 0x080808082b2b1919, 0x080808082b2b2b2b, 0x0808081908080819,- 0x0808081908081908, 0x080808190808192b, 0x0808081908082b19, 0x0808081908190808, 0x080808190819082b,- 0x0808081908191919, 0x0808081908192b08, 0x08080819082b0819, 0x08080819082b1908, 0x0808081919080808,- 0x080808191908082b, 0x0808081919081919, 0x0808081919082b08, 0x0808081919190819, 0x0808081919191908,- 0x080808191919192b, 0x0808081919192b19, 0x08080819192b0808, 0x08080819192b1919, 0x08080819192b2b08,- 0x080808192b080819, 0x080808192b081908, 0x080808192b190808, 0x080808192b19082b, 0x080808192b191919,- 0x080808192b2b0819, 0x080808192b2b1908, 0x0808082b08080808, 0x0808082b0808082b, 0x0808082b08081919,- 0x0808082b08082b08, 0x0808082b08190819, 0x0808082b08191908, 0x0808082b082b0808, 0x0808082b082b2b2b,- 0x0808082b19080819, 0x0808082b19081908, 0x0808082b1908192b, 0x0808082b19082b19, 0x0808082b19190808,- 0x0808082b19191919, 0x0808082b2b080808, 0x0808082b2b081919, 0x0808082b2b082b2b, 0x0808082b2b191908,- 0x0808082b2b2b082b, 0x0808190808080819, 0x0808190808081908, 0x080819080808192b, 0x0808190808082b19,- 0x0808190808190808, 0x080819080819082b, 0x0808190808191919, 0x0808190808192b08, 0x08081908082b0819,- 0x08081908082b1908, 0x08081908082b192b, 0x08081908082b2b19, 0x0808190819080808, 0x080819081908082b,- 0x0808190819081919, 0x0808190819082b08, 0x0808190819082b2b, 0x0808190819190819, 0x0808190819191908,- 0x080819081919192b, 0x0808190819192b19, 0x08081908192b0808, 0x08081908192b082b, 0x08081908192b1919,- 0x080819082b080819, 0x080819082b081908, 0x080819082b08192b, 0x080819082b082b19, 0x080819082b190808,- 0x080819082b191919, 0x080819082b192b08, 0x080819082b2b0819, 0x080819082b2b1908, 0x0808191908080808,- 0x080819190808082b, 0x0808191908081919, 0x0808191908082b08, 0x0808191908082b2b, 0x0808191908190819,- 0x0808191908191908, 0x080819190819192b, 0x0808191908192b19, 0x08081919082b0808, 0x08081919082b1919,- 0x08081919082b2b08, 0x0808191919080819, 0x0808191919081908, 0x080819191908192b, 0x0808191919082b19,- 0x0808191919190808, 0x080819191919082b, 0x0808191919191919, 0x0808191919192b08, 0x08081919192b0819,- 0x08081919192b1908, 0x080819192b080808, 0x080819192b08082b, 0x080819192b081919, 0x080819192b082b08,- 0x080819192b190819, 0x080819192b191908, 0x080819192b2b0808, 0x0808192b08080819, 0x0808192b08081908,- 0x0808192b0808192b, 0x0808192b08082b19, 0x0808192b08190808, 0x0808192b08191919, 0x0808192b19080808,- 0x0808192b19081919, 0x0808192b19082b08, 0x0808192b19190819, 0x0808192b19191908, 0x0808192b192b0808,- 0x0808192b2b080819, 0x0808192b2b081908, 0x0808192b2b190808, 0x08082b0808080808, 0x08082b080808082b,- 0x08082b0808081919, 0x08082b0808082b08, 0x08082b0808190819, 0x08082b0808191908, 0x08082b080819192b,- 0x08082b0808192b19, 0x08082b08082b0808, 0x08082b08082b1919, 0x08082b08082b2b2b, 0x08082b0819080819,- 0x08082b0819081908, 0x08082b081908192b, 0x08082b0819082b19, 0x08082b0819190808, 0x08082b081919082b,- 0x08082b0819191919, 0x08082b0819192b08, 0x08082b08192b0819, 0x08082b08192b1908, 0x08082b082b080808,- 0x08082b082b081919, 0x08082b082b191908, 0x08082b082b2b2b2b, 0x08082b1908080819, 0x08082b1908081908,- 0x08082b1908190808, 0x08082b190819082b, 0x08082b1908191919, 0x08082b1908192b08, 0x08082b19082b0819,- 0x08082b1919080808, 0x08082b1919081919, 0x08082b1919082b08, 0x08082b1919190819, 0x08082b1919191908,- 0x08082b19192b0808, 0x08082b192b080819, 0x08082b192b190808, 0x08082b2b08080808, 0x08082b2b08190819,- 0x08082b2b08191908, 0x08082b2b082b082b, 0x08082b2b082b2b08, 0x08082b2b082b2b2b, 0x08082b2b19190808,- 0x08082b2b2b192b19, 0x0819080808080819, 0x0819080808081908, 0x081908080808192b, 0x0819080808082b19,- 0x0819080808190808, 0x081908080819082b, 0x0819080808191919, 0x0819080808192b08, 0x08190808082b0819,- 0x08190808082b1908, 0x08190808082b192b, 0x0819080819080808, 0x081908081908082b, 0x0819080819081919,- 0x0819080819082b08, 0x0819080819190819, 0x0819080819191908, 0x081908081919192b, 0x0819080819192b19,- 0x08190808192b0808, 0x08190808192b082b, 0x08190808192b1919, 0x08190808192b2b08, 0x081908082b080819,- 0x081908082b081908, 0x081908082b08192b, 0x081908082b190808, 0x081908082b191919, 0x081908082b192b08,- 0x081908082b2b0819, 0x081908082b2b1908, 0x0819081908080808, 0x081908190808082b, 0x0819081908081919,- 0x0819081908082b08, 0x0819081908082b2b, 0x0819081908190819, 0x0819081908191908, 0x081908190819192b,- 0x0819081908192b19, 0x08190819082b0808, 0x08190819082b082b, 0x08190819082b1919, 0x08190819082b2b08,- 0x0819081919080819, 0x0819081919081908, 0x081908191908192b, 0x0819081919082b19, 0x0819081919190808,- 0x081908191919082b, 0x0819081919191919, 0x0819081919192b08, 0x08190819192b0819, 0x08190819192b1908,- 0x081908192b080808, 0x081908192b08082b, 0x081908192b081919, 0x081908192b082b08, 0x081908192b190819,- 0x081908192b191908, 0x0819082b08080819, 0x0819082b08081908, 0x0819082b08082b19, 0x0819082b08190808,- 0x0819082b08191919, 0x0819082b082b0819, 0x0819082b082b1908, 0x0819082b19080808, 0x0819082b19081919,- 0x0819082b19190819, 0x0819082b19191908, 0x0819082b2b080819, 0x0819082b2b081908, 0x0819082b2b190808,- 0x0819190808080808, 0x081919080808082b, 0x0819190808081919, 0x0819190808082b08, 0x0819190808190819,- 0x0819190808191908, 0x081919080819192b, 0x0819190808192b19, 0x08191908082b0808, 0x08191908082b1919,- 0x08191908082b2b08, 0x0819190819080819, 0x0819190819081908, 0x081919081908192b, 0x0819190819082b19,- 0x0819190819190808, 0x081919081919082b, 0x0819190819191919, 0x0819190819192b08, 0x08191908192b0819,- 0x08191908192b1908, 0x081919082b080808, 0x081919082b08082b, 0x081919082b081919, 0x081919082b082b08,- 0x081919082b190819, 0x081919082b191908, 0x081919082b2b0808, 0x0819191908080819, 0x0819191908081908,- 0x081919190808192b, 0x0819191908082b19, 0x0819191908190808, 0x081919190819082b, 0x0819191908191919,- 0x0819191908192b08, 0x08191919082b0819, 0x08191919082b1908, 0x0819191919080808, 0x081919191908082b,- 0x0819191919081919, 0x0819191919082b08, 0x0819191919190819, 0x0819191919191908, 0x08191919192b0808,- 0x081919192b080819, 0x081919192b081908, 0x081919192b190808, 0x0819192b08080808, 0x0819192b08081919,- 0x0819192b08082b08, 0x0819192b08190819, 0x0819192b08191908, 0x0819192b082b0808, 0x0819192b19080819,- 0x0819192b19081908, 0x0819192b19190808, 0x0819192b2b080808, 0x0819192b2b2b2b2b, 0x08192b0808080819,- 0x08192b0808081908, 0x08192b080808192b, 0x08192b0808082b19, 0x08192b0808190808, 0x08192b0808191919,- 0x08192b0808192b08, 0x08192b08082b0819, 0x08192b0819080808, 0x08192b081908082b, 0x08192b0819081919,- 0x08192b0819082b08, 0x08192b0819190819, 0x08192b0819191908, 0x08192b08192b0808, 0x08192b082b080819,- 0x08192b082b081908, 0x08192b1908080808, 0x08192b190808082b, 0x08192b1908081919, 0x08192b1908082b08,- 0x08192b1908190819, 0x08192b1908191908, 0x08192b19082b0808, 0x08192b1919080819, 0x08192b1919081908,- 0x08192b1919190808, 0x08192b19192b2b19, 0x08192b192b2b082b, 0x08192b2b08081908, 0x08192b2b08190808,- 0x08192b2b19080808, 0x08192b2b1919192b, 0x082b080808080808, 0x082b08080808082b, 0x082b080808081919,- 0x082b080808082b08, 0x082b080808190819, 0x082b080808191908, 0x082b08080819192b, 0x082b080808192b19,- 0x082b0808082b0808, 0x082b0808082b1919, 0x082b0808082b2b2b, 0x082b080819080819, 0x082b080819081908,- 0x082b080819190808, 0x082b08081919082b, 0x082b080819191919, 0x082b0808192b1908, 0x082b08082b080808,- 0x082b08082b082b2b, 0x082b08082b191908, 0x082b08082b2b2b2b, 0x082b081908080819, 0x082b081908081908,- 0x082b081908190808, 0x082b08190819082b, 0x082b081908191919, 0x082b0819082b0819, 0x082b081919080808,- 0x082b08191908082b, 0x082b081919081919, 0x082b081919190819, 0x082b081919191908, 0x082b0819192b0808,- 0x082b08192b080819, 0x082b08192b081908, 0x082b08192b190808, 0x082b082b08080808, 0x082b082b08082b2b,- 0x082b082b082b082b, 0x082b082b082b2b08, 0x082b082b082b2b2b, 0x082b082b19081908, 0x082b082b19190808,- 0x082b082b2b082b08, 0x082b082b2b082b2b, 0x082b082b2b2b2b08, 0x082b190808080819, 0x082b190808081908,- 0x082b19080808192b, 0x082b190808082b19, 0x082b190808190808, 0x082b190808191919, 0x082b190808192b08,- 0x082b1908082b0819, 0x082b1908082b1908, 0x082b190819080808, 0x082b19081908082b, 0x082b190819081919,- 0x082b190819082b08, 0x082b190819190819, 0x082b190819191908, 0x082b1908192b0808, 0x082b19082b080819,- 0x082b19082b081908, 0x082b19082b190808, 0x082b191908080808, 0x082b191908081919, 0x082b191908082b08,- 0x082b191908190819, 0x082b191908191908, 0x082b1919082b0808, 0x082b191919080819, 0x082b191919081908,- 0x082b191919190808, 0x082b1919192b192b, 0x082b19192b080808, 0x082b192b08080819, 0x082b192b08081908,- 0x082b192b08190808, 0x082b192b19080808, 0x082b192b19192b19, 0x082b2b0808080808, 0x082b2b0808081919,- 0x082b2b0808190819, 0x082b2b0808191908, 0x082b2b0819080819, 0x082b2b0819081908, 0x082b2b0819190808,- 0x082b2b082b082b2b, 0x082b2b082b2b2b2b, 0x082b2b1908080819, 0x082b2b1908081908, 0x082b2b1908190808,- 0x082b2b192b191919, 0x082b2b2b08082b2b, 0x082b2b2b082b082b, 0x082b2b2b192b1908, 0x082b2b2b2b082b08,- 0x082b2b2b2b082b2b, 0x1908080808080819, 0x1908080808081908, 0x190808080808192b, 0x1908080808082b19,- 0x1908080808190808, 0x190808080819082b, 0x1908080808191919, 0x1908080808192b08, 0x1908080808192b2b,- 0x19080808082b0819, 0x19080808082b1908, 0x19080808082b192b, 0x1908080819080808, 0x190808081908082b,- 0x1908080819081919, 0x1908080819082b08, 0x1908080819082b2b, 0x1908080819190819, 0x1908080819191908,- 0x190808081919192b, 0x1908080819192b19, 0x19080808192b0808, 0x19080808192b082b, 0x19080808192b1919,- 0x190808082b080819, 0x190808082b081908, 0x190808082b190808, 0x190808082b191919, 0x190808082b192b08,- 0x190808082b2b0819, 0x190808082b2b1908, 0x1908081908080808, 0x190808190808082b, 0x1908081908081919,- 0x1908081908082b08, 0x1908081908190819, 0x1908081908191908, 0x190808190819192b, 0x1908081908192b19,- 0x19080819082b0808, 0x19080819082b082b, 0x19080819082b1919, 0x1908081919080819, 0x1908081919081908,- 0x190808191908192b, 0x1908081919082b19, 0x1908081919190808, 0x190808191919082b, 0x1908081919191919,- 0x1908081919192b08, 0x19080819192b0819, 0x19080819192b1908, 0x190808192b080808, 0x190808192b08082b,- 0x190808192b081919, 0x190808192b082b08, 0x190808192b190819, 0x190808192b191908, 0x190808192b2b0808,- 0x1908082b08080819, 0x1908082b08081908, 0x1908082b08190808, 0x1908082b0819082b, 0x1908082b08191919,- 0x1908082b08192b08, 0x1908082b082b1908, 0x1908082b19080808, 0x1908082b19081919, 0x1908082b19082b08,- 0x1908082b19190819, 0x1908082b19191908, 0x1908082b192b0808, 0x1908082b2b080819, 0x1908082b2b081908,- 0x1908190808080808, 0x190819080808082b, 0x1908190808081919, 0x1908190808082b08, 0x1908190808082b2b,- 0x1908190808190819, 0x1908190808191908, 0x190819080819192b, 0x1908190808192b19, 0x19081908082b0808,- 0x19081908082b082b, 0x19081908082b1919, 0x19081908082b2b08, 0x1908190819080819, 0x1908190819081908,- 0x190819081908192b, 0x1908190819082b19, 0x1908190819190808, 0x190819081919082b, 0x1908190819191919,- 0x1908190819192b08, 0x19081908192b0819, 0x19081908192b1908, 0x190819082b080808, 0x190819082b08082b,- 0x190819082b081919, 0x190819082b082b08, 0x190819082b190819, 0x190819082b191908, 0x190819082b2b0808,- 0x1908191908080819, 0x1908191908081908, 0x190819190808192b, 0x1908191908082b19, 0x1908191908190808,- 0x190819190819082b, 0x1908191908191919, 0x1908191908192b08, 0x19081919082b0819, 0x19081919082b1908,- 0x1908191919080808, 0x190819191908082b, 0x1908191919081919, 0x1908191919082b08, 0x1908191919190819,- 0x1908191919191908, 0x19081919192b0808, 0x19081919192b2b2b, 0x190819192b080819, 0x190819192b081908,- 0x190819192b190808, 0x1908192b08080808, 0x1908192b0808082b, 0x1908192b08081919, 0x1908192b08082b08,- 0x1908192b08190819, 0x1908192b08191908, 0x1908192b082b0808, 0x1908192b19080819, 0x1908192b19081908,- 0x1908192b19190808, 0x1908192b2b080808, 0x1908192b2b2b1919, 0x19082b0808080819, 0x19082b0808081908,- 0x19082b0808082b19, 0x19082b0808190808, 0x19082b080819082b, 0x19082b0808191919, 0x19082b0808192b08,- 0x19082b08082b0819, 0x19082b08082b1908, 0x19082b0819080808, 0x19082b081908082b, 0x19082b0819081919,- 0x19082b0819082b08, 0x19082b0819190819, 0x19082b0819191908, 0x19082b08192b0808, 0x19082b082b081908,- 0x19082b082b190808, 0x19082b1908080808, 0x19082b190808082b, 0x19082b1908081919, 0x19082b1908082b08,- 0x19082b1908190819, 0x19082b1908191908, 0x19082b19082b0808, 0x19082b1919080819, 0x19082b1919081908,- 0x19082b1919190808, 0x19082b192b080808, 0x19082b192b19192b, 0x19082b2b08080819, 0x19082b2b08081908,- 0x19082b2b08190808, 0x19082b2b19080808, 0x1919080808080808, 0x191908080808082b, 0x1919080808081919,- 0x1919080808082b08, 0x1919080808190819, 0x1919080808191908, 0x191908080819192b, 0x1919080808192b19,- 0x19190808082b0808, 0x19190808082b082b, 0x19190808082b1919, 0x19190808082b2b08, 0x1919080819080819,- 0x1919080819081908, 0x191908081908192b, 0x1919080819082b19, 0x1919080819190808, 0x191908081919082b,- 0x1919080819191919, 0x1919080819192b08, 0x19190808192b0819, 0x19190808192b1908, 0x191908082b080808,- 0x191908082b08082b, 0x191908082b081919, 0x191908082b082b08, 0x191908082b190819, 0x191908082b191908,- 0x1919081908080819, 0x1919081908081908, 0x191908190808192b, 0x1919081908082b19, 0x1919081908190808,- 0x191908190819082b, 0x1919081908191919, 0x1919081908192b08, 0x19190819082b0819, 0x19190819082b1908,- 0x1919081919080808, 0x191908191908082b, 0x1919081919081919, 0x1919081919082b08, 0x1919081919190819,- 0x1919081919191908, 0x19190819192b0808, 0x191908192b080819, 0x191908192b081908, 0x191908192b190808,- 0x1919082b08080808, 0x1919082b08081919, 0x1919082b08082b08, 0x1919082b08190819, 0x1919082b08191908,- 0x1919082b082b0808, 0x1919082b19080819, 0x1919082b19081908, 0x1919082b19190808, 0x1919082b192b2b19,- 0x1919082b2b080808, 0x1919190808080819, 0x1919190808081908, 0x191919080808192b, 0x1919190808082b19,- 0x1919190808190808, 0x191919080819082b, 0x1919190808191919, 0x1919190808192b08, 0x19191908082b0819,- 0x19191908082b1908, 0x1919190819080808, 0x191919081908082b, 0x1919190819081919, 0x1919190819082b08,- 0x1919190819190819, 0x1919190819191908, 0x19191908192b0808, 0x191919082b080819, 0x191919082b081908,- 0x191919082b190808, 0x1919191908080808, 0x191919190808082b, 0x1919191908081919, 0x1919191908082b08,- 0x1919191908190819, 0x1919191908191908, 0x19191919082b0808, 0x1919191919080819, 0x1919191919081908,- 0x1919191919190808, 0x191919192b080808, 0x1919192b08080819, 0x1919192b08081908, 0x1919192b08190808,- 0x1919192b082b192b, 0x1919192b19080808, 0x19192b0808080808, 0x19192b080808082b, 0x19192b0808081919,- 0x19192b0808082b08, 0x19192b0808190819, 0x19192b0808191908, 0x19192b08082b0808, 0x19192b0819080819,- 0x19192b0819081908, 0x19192b0819190808, 0x19192b0819192b2b, 0x19192b082b080808, 0x19192b1908080819,- 0x19192b1908081908, 0x19192b1908190808, 0x19192b1919080808, 0x19192b2b08080808, 0x19192b2b08192b19,- 0x19192b2b2b081919, 0x19192b2b2b2b2b08, 0x192b080808080819, 0x192b080808081908, 0x192b08080808192b,- 0x192b080808190808, 0x192b08080819082b, 0x192b080808191919, 0x192b080808192b08, 0x192b0808082b0819,- 0x192b0808082b1908, 0x192b080819080808, 0x192b080819081919, 0x192b080819082b08, 0x192b080819190819,- 0x192b080819191908, 0x192b0808192b0808, 0x192b08082b081908, 0x192b08082b190808, 0x192b081908080808,- 0x192b08190808082b, 0x192b081908081919, 0x192b081908082b08, 0x192b081908190819, 0x192b081908191908,- 0x192b0819082b0808, 0x192b081919080819, 0x192b081919081908, 0x192b081919190808, 0x192b08192b080808,- 0x192b08192b192b19, 0x192b082b08081908, 0x192b082b08190808, 0x192b082b19080808, 0x192b082b1919192b,- 0x192b082b2b2b0819, 0x192b190808080808, 0x192b190808081919, 0x192b190808082b08, 0x192b190808190819,- 0x192b190808191908, 0x192b1908082b0808, 0x192b190819080819, 0x192b190819081908, 0x192b190819190808,- 0x192b19082b080808, 0x192b191908080819, 0x192b191908081908, 0x192b191908190808, 0x192b191919080808,- 0x192b191919082b2b, 0x192b1919192b2b08, 0x192b19192b19082b, 0x192b192b08080808, 0x192b192b2b191908,- 0x192b2b0808080819, 0x192b2b0808081908, 0x192b2b0808190808, 0x192b2b08192b1919, 0x192b2b082b192b08,- 0x192b2b1908080808, 0x192b2b19082b2b2b, 0x192b2b2b1908082b, 0x192b2b2b2b2b0819, 0x2b08080808080808,- 0x2b0808080808082b, 0x2b08080808081919, 0x2b08080808082b08, 0x2b08080808190819, 0x2b08080808191908,- 0x2b08080808192b19, 0x2b080808082b0808, 0x2b080808082b1919, 0x2b08080819080819, 0x2b08080819081908,- 0x2b08080819190808, 0x2b0808081919082b, 0x2b08080819191919, 0x2b08080819192b08, 0x2b080808192b0819,- 0x2b0808082b080808, 0x2b0808082b081919, 0x2b0808082b190819, 0x2b0808082b191908, 0x2b08081908080819,- 0x2b08081908081908, 0x2b08081908082b19, 0x2b08081908190808, 0x2b0808190819082b, 0x2b08081908191919,- 0x2b08081908192b08, 0x2b080819082b0819, 0x2b080819082b1908, 0x2b08081919080808, 0x2b0808191908082b,- 0x2b08081919081919, 0x2b08081919082b08, 0x2b08081919190819, 0x2b08081919191908, 0x2b0808192b080819,- 0x2b0808192b081908, 0x2b0808192b190808, 0x2b0808192b2b2b19, 0x2b08082b08080808, 0x2b08082b08081919,- 0x2b08082b08082b2b, 0x2b08082b08190819, 0x2b08082b08191908, 0x2b08082b19080819, 0x2b08082b19081908,- 0x2b08082b19190808, 0x2b08190808080819, 0x2b08190808081908, 0x2b0819080808192b, 0x2b08190808082b19,- 0x2b08190808190808, 0x2b0819080819082b, 0x2b08190808191919, 0x2b08190808192b08, 0x2b081908082b0819,- 0x2b08190819080808, 0x2b0819081908082b, 0x2b08190819081919, 0x2b08190819082b08, 0x2b08190819190819,- 0x2b08190819191908, 0x2b081908192b0808, 0x2b0819082b080819, 0x2b0819082b081908, 0x2b0819082b190808,- 0x2b08191908080808, 0x2b0819190808082b, 0x2b08191908081919, 0x2b08191908082b08, 0x2b08191908190819,- 0x2b08191908191908, 0x2b081919082b0808, 0x2b08191919080819, 0x2b08191919081908, 0x2b08191919190808,- 0x2b0819192b080808, 0x2b0819192b082b2b, 0x2b08192b08080819, 0x2b08192b08081908, 0x2b08192b08190808,- 0x2b08192b082b2b19, 0x2b08192b19080808, 0x2b082b0808080808, 0x2b082b0808081919, 0x2b082b0808190819,- 0x2b082b0808191908, 0x2b082b0819080819, 0x2b082b0819081908, 0x2b082b0819190808, 0x2b082b082b2b082b,- 0x2b082b1908080819, 0x2b082b1908081908, 0x2b082b1919080808, 0x2b082b19192b1919, 0x2b082b2b082b082b,- 0x2b082b2b19192b08, 0x2b082b2b19192b2b, 0x2b082b2b2b08082b, 0x2b082b2b2b2b082b, 0x2b19080808080819,- 0x2b19080808081908, 0x2b19080808082b19, 0x2b19080808190808, 0x2b1908080819082b, 0x2b19080808191919,- 0x2b19080808192b08, 0x2b190808082b1908, 0x2b19080819080808, 0x2b1908081908082b, 0x2b19080819081919,- 0x2b19080819082b08, 0x2b19080819190819, 0x2b19080819191908, 0x2b190808192b0808, 0x2b1908082b080819,- 0x2b1908082b081908, 0x2b1908082b190808, 0x2b19081908080808, 0x2b19081908081919, 0x2b19081908190819,- 0x2b19081908191908, 0x2b19081919080819, 0x2b19081919081908, 0x2b19081919190808, 0x2b19081919192b2b,- 0x2b19082b08080819, 0x2b19082b08081908, 0x2b19082b08190808, 0x2b19082b19080808, 0x2b19082b2b2b192b,- 0x2b19190808080808, 0x2b1919080808082b, 0x2b19190808081919, 0x2b19190808082b08, 0x2b19190808190819,- 0x2b19190808191908, 0x2b191908082b0808, 0x2b19190819080819, 0x2b19190819081908, 0x2b19190819190808,- 0x2b1919082b080808, 0x2b1919082b19192b, 0x2b19191908080819, 0x2b19191908081908, 0x2b19191908190808,- 0x2b19191919080808, 0x2b1919192b192b08, 0x2b1919192b2b0819, 0x2b19192b08080808, 0x2b19192b1908192b,- 0x2b19192b192b1908, 0x2b192b0808080819, 0x2b192b0808081908, 0x2b192b0808190808, 0x2b192b08082b192b,- 0x2b192b0819080808, 0x2b192b082b2b2b19, 0x2b192b1908080808, 0x2b192b1919082b19, 0x2b192b191919082b,- 0x2b192b2b2b190808, 0x2b2b080808080808, 0x2b2b080808081919, 0x2b2b080808082b2b, 0x2b2b080808191908,- 0x2b2b0808082b082b, 0x2b2b0808082b2b2b, 0x2b2b080819080819, 0x2b2b080819081908, 0x2b2b080819190808,- 0x2b2b08082b2b082b, 0x2b2b08082b2b2b2b, 0x2b2b081919080808, 0x2b2b0819192b1919, 0x2b2b082b0808082b,- 0x2b2b082b08082b2b, 0x2b2b082b082b082b, 0x2b2b082b082b2b08, 0x2b2b082b082b2b2b, 0x2b2b082b2b08082b,- 0x2b2b082b2b082b08, 0x2b2b082b2b082b2b, 0x2b2b082b2b2b2b08, 0x2b2b190808080819, 0x2b2b190808081908,- 0x2b2b190808190808, 0x2b2b190819080808, 0x2b2b19082b082b19, 0x2b2b19082b2b1908, 0x2b2b191908080808,- 0x2b2b191908192b19, 0x2b2b192b19190819, 0x2b2b2b0808082b2b, 0x2b2b2b08082b2b08, 0x2b2b2b082b2b082b,- 0x2b2b2b1919191908, 0x2b2b2b192b08192b, 0x2b2b2b2b08082b08, 0x2b2b2b2b08082b2b, 0x2b2b2b2b082b0808,- 0x2b2b2b2b082b082b, 0x2b2b2b2b082b2b08, 0x2b2b2b2b2b082b08, 0x2b2b2b2b2b2b2b2b,-};--static const __device__ uint32_t iq3xxs_grid[256] = {- 0x04040404, 0x04040414, 0x04040424, 0x04040c0c, 0x04040c1c, 0x04040c3e, 0x04041404, 0x04041414, 0x04041c0c,- 0x04042414, 0x04043e1c, 0x04043e2c, 0x040c040c, 0x040c041c, 0x040c0c04, 0x040c0c14, 0x040c140c, 0x040c142c,- 0x040c1c04, 0x040c1c14, 0x040c240c, 0x040c2c24, 0x040c3e04, 0x04140404, 0x04140414, 0x04140424, 0x04140c0c,- 0x04141404, 0x04141414, 0x04141c0c, 0x04141c1c, 0x04141c3e, 0x04142c0c, 0x04142c3e, 0x04143e2c, 0x041c040c,- 0x041c043e, 0x041c0c04, 0x041c0c14, 0x041c142c, 0x041c3e04, 0x04240c1c, 0x04241c3e, 0x04242424, 0x04242c3e,- 0x04243e1c, 0x04243e2c, 0x042c040c, 0x042c043e, 0x042c1c14, 0x042c2c14, 0x04341c2c, 0x04343424, 0x043e0c04,- 0x043e0c24, 0x043e0c34, 0x043e241c, 0x043e340c, 0x0c04040c, 0x0c04041c, 0x0c040c04, 0x0c040c14, 0x0c04140c,- 0x0c04141c, 0x0c041c04, 0x0c041c14, 0x0c041c24, 0x0c04243e, 0x0c042c04, 0x0c0c0404, 0x0c0c0414, 0x0c0c0c0c,- 0x0c0c1404, 0x0c0c1414, 0x0c14040c, 0x0c14041c, 0x0c140c04, 0x0c140c14, 0x0c14140c, 0x0c141c04, 0x0c143e14,- 0x0c1c0404, 0x0c1c0414, 0x0c1c1404, 0x0c1c1c0c, 0x0c1c2434, 0x0c1c3434, 0x0c24040c, 0x0c24042c, 0x0c242c04,- 0x0c2c1404, 0x0c2c1424, 0x0c2c2434, 0x0c2c3e0c, 0x0c34042c, 0x0c3e1414, 0x0c3e2404, 0x14040404, 0x14040414,- 0x14040c0c, 0x14040c1c, 0x14041404, 0x14041414, 0x14041434, 0x14041c0c, 0x14042414, 0x140c040c, 0x140c041c,- 0x140c042c, 0x140c0c04, 0x140c0c14, 0x140c140c, 0x140c1c04, 0x140c341c, 0x140c343e, 0x140c3e04, 0x14140404,- 0x14140414, 0x14140c0c, 0x14140c3e, 0x14141404, 0x14141414, 0x14141c3e, 0x14142404, 0x14142c2c, 0x141c040c,- 0x141c0c04, 0x141c0c24, 0x141c3e04, 0x141c3e24, 0x14241c2c, 0x14242c1c, 0x142c041c, 0x142c143e, 0x142c240c,- 0x142c3e24, 0x143e040c, 0x143e041c, 0x143e0c34, 0x143e242c, 0x1c04040c, 0x1c040c04, 0x1c040c14, 0x1c04140c,- 0x1c04141c, 0x1c042c04, 0x1c04342c, 0x1c043e14, 0x1c0c0404, 0x1c0c0414, 0x1c0c1404, 0x1c0c1c0c, 0x1c0c2424,- 0x1c0c2434, 0x1c14040c, 0x1c14041c, 0x1c140c04, 0x1c14142c, 0x1c142c14, 0x1c143e14, 0x1c1c0c0c, 0x1c1c1c1c,- 0x1c241c04, 0x1c24243e, 0x1c243e14, 0x1c2c0404, 0x1c2c0434, 0x1c2c1414, 0x1c2c2c2c, 0x1c340c24, 0x1c341c34,- 0x1c34341c, 0x1c3e1c1c, 0x1c3e3404, 0x24040424, 0x24040c3e, 0x24041c2c, 0x24041c3e, 0x24042c1c, 0x24042c3e,- 0x240c3e24, 0x24141404, 0x24141c3e, 0x24142404, 0x24143404, 0x24143434, 0x241c043e, 0x241c242c, 0x24240424,- 0x24242c0c, 0x24243424, 0x242c142c, 0x242c241c, 0x242c3e04, 0x243e042c, 0x243e0c04, 0x243e0c14, 0x243e1c04,- 0x2c040c14, 0x2c04240c, 0x2c043e04, 0x2c0c0404, 0x2c0c0434, 0x2c0c1434, 0x2c0c2c2c, 0x2c140c24, 0x2c141c14,- 0x2c143e14, 0x2c1c0414, 0x2c1c2c1c, 0x2c240c04, 0x2c24141c, 0x2c24143e, 0x2c243e14, 0x2c2c0414, 0x2c2c1c0c,- 0x2c342c04, 0x2c3e1424, 0x2c3e2414, 0x34041424, 0x34042424, 0x34042434, 0x34043424, 0x340c140c, 0x340c340c,- 0x34140c3e, 0x34143424, 0x341c1c04, 0x341c1c34, 0x34242424, 0x342c042c, 0x342c2c14, 0x34341c1c, 0x343e041c,- 0x343e140c, 0x3e04041c, 0x3e04042c, 0x3e04043e, 0x3e040c04, 0x3e041c14, 0x3e042c14, 0x3e0c1434, 0x3e0c2404,- 0x3e140c14, 0x3e14242c, 0x3e142c14, 0x3e1c0404, 0x3e1c0c2c, 0x3e1c1c1c, 0x3e1c3404, 0x3e24140c, 0x3e24240c,- 0x3e2c0404, 0x3e2c0414, 0x3e2c1424, 0x3e341c04,-};--static const __device__ uint32_t iq3xs_grid[512] = {- 0x04040404, 0x0404040c, 0x04040414, 0x0404042c, 0x0404043e, 0x04040c04, 0x04040c0c, 0x04040c14, 0x04040c24,- 0x04040c34, 0x04041404, 0x0404140c, 0x0404142c, 0x04041c1c, 0x04042404, 0x04042414, 0x0404242c, 0x0404243e,- 0x04042c0c, 0x04042c1c, 0x04043404, 0x04043414, 0x04043e0c, 0x04043e24, 0x04043e3e, 0x040c0404, 0x040c040c,- 0x040c0414, 0x040c0424, 0x040c0c04, 0x040c0c0c, 0x040c0c2c, 0x040c1404, 0x040c141c, 0x040c143e, 0x040c1c0c,- 0x040c1c2c, 0x040c2424, 0x040c340c, 0x040c342c, 0x040c3e14, 0x04140404, 0x0414040c, 0x0414042c, 0x0414043e,- 0x04140c04, 0x04140c1c, 0x04140c34, 0x0414140c, 0x0414142c, 0x04141c04, 0x04141c24, 0x04142414, 0x0414242c,- 0x0414243e, 0x04142c0c, 0x04142c1c, 0x04143e04, 0x04143e1c, 0x041c041c, 0x041c0c0c, 0x041c0c2c, 0x041c1404,- 0x041c1414, 0x041c1c0c, 0x041c1c1c, 0x041c1c34, 0x041c2424, 0x041c2c04, 0x041c2c14, 0x041c343e, 0x041c3e0c,- 0x041c3e2c, 0x04240404, 0x04240c1c, 0x04240c3e, 0x0424140c, 0x04241424, 0x04241c14, 0x04242404, 0x0424241c,- 0x04242c0c, 0x04243e04, 0x042c0414, 0x042c0424, 0x042c1404, 0x042c1414, 0x042c1434, 0x042c1c1c, 0x042c240c,- 0x042c242c, 0x042c243e, 0x042c3434, 0x042c3e1c, 0x04340434, 0x04340c0c, 0x04340c1c, 0x04341c0c, 0x04342c14,- 0x04343e0c, 0x043e0404, 0x043e0414, 0x043e0424, 0x043e1404, 0x043e1414, 0x043e1434, 0x043e1c1c, 0x043e2c04,- 0x043e2c24, 0x0c040404, 0x0c04040c, 0x0c040414, 0x0c040424, 0x0c040c04, 0x0c040c0c, 0x0c040c1c, 0x0c040c2c,- 0x0c040c3e, 0x0c041404, 0x0c041414, 0x0c041c0c, 0x0c041c24, 0x0c041c34, 0x0c042c24, 0x0c042c34, 0x0c04340c,- 0x0c043e14, 0x0c0c0404, 0x0c0c040c, 0x0c0c041c, 0x0c0c0434, 0x0c0c0c04, 0x0c0c0c24, 0x0c0c140c, 0x0c0c1c04,- 0x0c0c1c1c, 0x0c0c240c, 0x0c0c2c04, 0x0c0c2c14, 0x0c0c3e04, 0x0c0c3e34, 0x0c140404, 0x0c140c14, 0x0c140c2c,- 0x0c140c3e, 0x0c141404, 0x0c141424, 0x0c141c14, 0x0c142404, 0x0c14241c, 0x0c142c2c, 0x0c143404, 0x0c143e14,- 0x0c1c040c, 0x0c1c0424, 0x0c1c043e, 0x0c1c0c04, 0x0c1c0c1c, 0x0c1c140c, 0x0c1c143e, 0x0c1c1c04, 0x0c1c1c24,- 0x0c1c240c, 0x0c1c3414, 0x0c1c3e04, 0x0c24041c, 0x0c24042c, 0x0c240c14, 0x0c240c24, 0x0c241c0c, 0x0c241c1c,- 0x0c242414, 0x0c242434, 0x0c242c04, 0x0c242c24, 0x0c2c040c, 0x0c2c0c04, 0x0c2c0c1c, 0x0c2c140c, 0x0c2c1c04,- 0x0c2c1c14, 0x0c2c2c0c, 0x0c341404, 0x0c341424, 0x0c34143e, 0x0c342424, 0x0c342434, 0x0c3e040c, 0x0c3e041c,- 0x0c3e0c04, 0x0c3e0c14, 0x0c3e140c, 0x0c3e1c2c, 0x0c3e240c, 0x0c3e3414, 0x0c3e3e04, 0x14040404, 0x1404040c,- 0x1404041c, 0x1404042c, 0x1404043e, 0x14040c04, 0x14040c14, 0x14040c24, 0x14040c34, 0x1404140c, 0x1404141c,- 0x1404143e, 0x14041c04, 0x14041c14, 0x1404240c, 0x1404241c, 0x1404242c, 0x14042c04, 0x14042c14, 0x1404343e,- 0x14043e04, 0x14043e1c, 0x14043e2c, 0x140c0404, 0x140c0414, 0x140c0c04, 0x140c0c1c, 0x140c0c3e, 0x140c1414,- 0x140c142c, 0x140c1c0c, 0x140c1c24, 0x140c2414, 0x140c2c0c, 0x1414040c, 0x14140424, 0x1414043e, 0x1414140c,- 0x1414141c, 0x14141c04, 0x14141c3e, 0x1414240c, 0x14142c1c, 0x14142c3e, 0x14143e0c, 0x14143e24, 0x141c0404,- 0x141c0414, 0x141c042c, 0x141c0c0c, 0x141c1414, 0x141c1424, 0x141c1c0c, 0x141c1c1c, 0x141c2414, 0x141c2c04,- 0x141c3434, 0x1424040c, 0x1424043e, 0x14241404, 0x1424141c, 0x14241c14, 0x14241c2c, 0x1424240c, 0x14243e14,- 0x14243e2c, 0x142c0424, 0x142c0c0c, 0x142c1414, 0x142c1c3e, 0x142c2404, 0x142c2c1c, 0x142c3e04, 0x14340404,- 0x14340414, 0x1434043e, 0x1434140c, 0x14342c2c, 0x1434340c, 0x143e042c, 0x143e0c0c, 0x143e1434, 0x143e1c04,- 0x143e241c, 0x143e2c04, 0x1c040414, 0x1c040c0c, 0x1c040c1c, 0x1c040c2c, 0x1c040c3e, 0x1c041414, 0x1c041c0c,- 0x1c041c1c, 0x1c041c2c, 0x1c042414, 0x1c042424, 0x1c04243e, 0x1c042c0c, 0x1c04341c, 0x1c043e0c, 0x1c0c040c,- 0x1c0c041c, 0x1c0c042c, 0x1c0c0c24, 0x1c0c140c, 0x1c0c141c, 0x1c0c2404, 0x1c0c3404, 0x1c0c3e14, 0x1c0c3e34,- 0x1c140404, 0x1c140c14, 0x1c141404, 0x1c141c14, 0x1c141c24, 0x1c142c04, 0x1c1c040c, 0x1c1c0c04, 0x1c1c0c24,- 0x1c1c140c, 0x1c1c141c, 0x1c1c143e, 0x1c1c1c04, 0x1c1c240c, 0x1c1c241c, 0x1c1c243e, 0x1c1c2c2c, 0x1c1c3e1c,- 0x1c24041c, 0x1c240c0c, 0x1c240c34, 0x1c241414, 0x1c241c0c, 0x1c242c14, 0x1c243404, 0x1c243424, 0x1c2c040c,- 0x1c2c0c04, 0x1c2c0c14, 0x1c2c142c, 0x1c2c1c14, 0x1c2c2424, 0x1c2c2c34, 0x1c2c3e1c, 0x1c340c34, 0x1c34240c,- 0x1c3e040c, 0x1c3e041c, 0x1c3e1404, 0x1c3e1414, 0x1c3e1c2c, 0x24040404, 0x24040424, 0x24040c14, 0x24041404,- 0x24041424, 0x2404143e, 0x24041c14, 0x2404240c, 0x24042c04, 0x24043e04, 0x240c0414, 0x240c043e, 0x240c0c0c,- 0x240c0c1c, 0x240c1414, 0x240c1c04, 0x240c1c2c, 0x240c241c, 0x240c2c0c, 0x240c2c2c, 0x2414040c, 0x2414041c,- 0x24140c04, 0x24140c2c, 0x2414140c, 0x24141c1c, 0x24142404, 0x24142c3e, 0x24143414, 0x24143e04, 0x241c0424,- 0x241c0c0c, 0x241c0c1c, 0x241c1404, 0x241c1414, 0x241c1c0c, 0x241c1c2c, 0x24240404, 0x24240414, 0x24241424,- 0x24241c3e, 0x24242404, 0x24243e0c, 0x242c042c, 0x242c043e, 0x242c140c, 0x242c3414, 0x24340c1c, 0x24341c24,- 0x24343404, 0x243e0c04, 0x243e0c2c, 0x243e1c04, 0x243e241c, 0x243e2c0c, 0x2c040414, 0x2c040c04, 0x2c040c24,- 0x2c041414, 0x2c042404, 0x2c042424, 0x2c04243e, 0x2c042c14, 0x2c043434, 0x2c043e24, 0x2c0c040c, 0x2c0c041c,- 0x2c0c042c, 0x2c0c0c14, 0x2c0c140c, 0x2c0c1c14, 0x2c0c3e14, 0x2c140404, 0x2c140c0c, 0x2c14141c, 0x2c141c04,- 0x2c141c34, 0x2c142c1c, 0x2c1c0414, 0x2c1c043e, 0x2c1c0c04, 0x2c1c143e, 0x2c1c2424, 0x2c1c2c0c, 0x2c1c342c,- 0x2c1c3e1c, 0x2c24040c, 0x2c240424, 0x2c241404, 0x2c241c14, 0x2c242434, 0x2c2c0c14, 0x2c2c1434, 0x2c2c2c0c,- 0x2c2c2c1c, 0x2c342414, 0x2c3e0414, 0x2c3e0424, 0x2c3e1414, 0x34040c0c, 0x34040c1c, 0x34040c2c, 0x34041c0c,- 0x34041c1c, 0x34043404, 0x340c0404, 0x340c1404, 0x340c143e, 0x340c3424, 0x34140c14, 0x34141c24, 0x34142414,- 0x34142c2c, 0x34143414, 0x34143e04, 0x341c0404, 0x341c0c24, 0x341c140c, 0x341c2404, 0x3424142c, 0x3424241c,- 0x34243414, 0x342c0404, 0x342c041c, 0x342c1c24, 0x342c3404, 0x3434042c, 0x34342404, 0x343e0c0c, 0x343e0c1c,- 0x3e040404, 0x3e040424, 0x3e04043e, 0x3e041404, 0x3e041414, 0x3e041c34, 0x3e042404, 0x3e042c24, 0x3e043414,- 0x3e0c0414, 0x3e0c0c0c, 0x3e0c1424, 0x3e0c241c, 0x3e0c242c, 0x3e14040c, 0x3e140424, 0x3e140c04, 0x3e140c34,- 0x3e14140c, 0x3e141c04, 0x3e142c0c, 0x3e1c0414, 0x3e1c1c14, 0x3e1c1c2c, 0x3e1c2c1c, 0x3e24040c, 0x3e24042c,- 0x3e240c1c, 0x3e241404, 0x3e242c04, 0x3e2c1414, 0x3e2c2414, 0x3e340414, 0x3e341c0c, 0x3e3e0404,-};--#define IQ1S_DELTA 0.125f-#define IQ1M_DELTA 0.125f-static const __device__ uint64_t iq1s_grid_gpu[2048] = {- 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000, 0x00020002,- 0x00020200, 0x00020202, 0x01000101, 0x01010001, 0x01010100, 0x01010102, 0x01020101, 0x02000000, 0x02000002,- 0x02000200, 0x02000202, 0x02010101, 0x02020000, 0x02020002, 0x02020200, 0x02020202, 0x00000110, 0x00000111,- 0x00010011, 0x00010110, 0x00010112, 0x00010211, 0x00010212, 0x00020111, 0x01000011, 0x01000112, 0x01000211,- 0x01010012, 0x01010111, 0x01010212, 0x01020011, 0x01020110, 0x01020112, 0x01020210, 0x02000111, 0x02010011,- 0x02010110, 0x02010112, 0x02020111, 0x00000020, 0x00000022, 0x00000220, 0x00000222, 0x00010121, 0x00020020,- 0x00020022, 0x00020220, 0x00020222, 0x01000121, 0x01010021, 0x01010221, 0x01020120, 0x01020221, 0x02000020,- 0x02000022, 0x02000220, 0x02000222, 0x02010021, 0x02010121, 0x02010221, 0x02020020, 0x02020022, 0x02020220,- 0x02020222, 0x00011001, 0x00011100, 0x00011102, 0x00021101, 0x01001001, 0x01001201, 0x01011101, 0x01011202,- 0x01021100, 0x01021101, 0x02011001, 0x02011201, 0x02021101, 0x00001011, 0x00001110, 0x00001111, 0x00001112,- 0x00011111, 0x00011210, 0x00011212, 0x00021211, 0x01001010, 0x01001111, 0x01001212, 0x01011010, 0x01011011,- 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010, 0x01021012, 0x01021111, 0x01021210, 0x01021212,- 0x02001011, 0x02011011, 0x02011111, 0x02011210, 0x02011212, 0x02021011, 0x02021110, 0x02021111, 0x02021112,- 0x02021211, 0x00011120, 0x00011221, 0x01001021, 0x01001120, 0x01011020, 0x01011022, 0x01011121, 0x01011220,- 0x01021020, 0x01021021, 0x01021122, 0x01021221, 0x02001121, 0x02011021, 0x02011120, 0x02011221, 0x00002000,- 0x00002002, 0x00002200, 0x00002202, 0x00012101, 0x00022000, 0x00022002, 0x00022200, 0x00022202, 0x01002101,- 0x01012001, 0x01012102, 0x01022101, 0x02002000, 0x02002002, 0x02002200, 0x02002202, 0x02012101, 0x02022000,- 0x02022002, 0x02022200, 0x02022202, 0x00002111, 0x00012011, 0x00012110, 0x00012211, 0x00022110, 0x00022111,- 0x01002011, 0x01012010, 0x01012011, 0x01012111, 0x01022011, 0x01022110, 0x01022211, 0x02012011, 0x02012110,- 0x02012112, 0x02012211, 0x02022111, 0x00002020, 0x00002022, 0x00002220, 0x00002222, 0x00012121, 0x00022020,- 0x00022022, 0x00022220, 0x00022222, 0x01002121, 0x01012021, 0x01012221, 0x01022021, 0x01022121, 0x02002020,- 0x02002022, 0x02002121, 0x02002220, 0x02002222, 0x02012121, 0x02022020, 0x02022022, 0x02022220, 0x02022222,- 0x00110000, 0x00110001, 0x00110100, 0x00110201, 0x00120100, 0x00120101, 0x01100001, 0x01100100, 0x01110000,- 0x01110101, 0x01110200, 0x01120001, 0x01120100, 0x01120101, 0x01120201, 0x02110001, 0x02110100, 0x02110102,- 0x02120001, 0x02120101, 0x00100011, 0x00100110, 0x00100112, 0x00100211, 0x00110010, 0x00110012, 0x00110111,- 0x00110210, 0x00120011, 0x00120110, 0x00120211, 0x01100111, 0x01100212, 0x01110010, 0x01110011, 0x01110012,- 0x01110110, 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111, 0x02100110, 0x02110012, 0x02110111,- 0x02120011, 0x02120110, 0x00110021, 0x00110120, 0x00110122, 0x00120121, 0x01100020, 0x01100122, 0x01100221,- 0x01110022, 0x01110121, 0x01110220, 0x01110222, 0x01120120, 0x01120122, 0x02100121, 0x02110021, 0x02110120,- 0x02110122, 0x02120121, 0x00101001, 0x00101102, 0x00101201, 0x00111100, 0x00111101, 0x00111200, 0x00111201,- 0x00121001, 0x00121102, 0x01101001, 0x01101101, 0x01101102, 0x01101200, 0x01101202, 0x01111001, 0x01111100,- 0x01111101, 0x01111102, 0x01111201, 0x01121002, 0x01121101, 0x01121200, 0x02101100, 0x02101201, 0x02111000,- 0x02111100, 0x02111101, 0x02111200, 0x02111201, 0x02111202, 0x02121001, 0x02121100, 0x02121101, 0x02121201,- 0x00101012, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112, 0x00111211, 0x00121010,- 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110, 0x01101111, 0x01101112, 0x01111011,- 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211, 0x01111212, 0x01121011, 0x01121110, 0x01121111,- 0x01121112, 0x01121211, 0x02101010, 0x02101012, 0x02101110, 0x02101111, 0x02101210, 0x02101212, 0x02111010,- 0x02111011, 0x02111110, 0x02111111, 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111,- 0x00101021, 0x00101120, 0x00101121, 0x00101122, 0x00111121, 0x00111122, 0x00111220, 0x00111222, 0x00121021,- 0x00121122, 0x01101020, 0x01101022, 0x01101120, 0x01101121, 0x01101220, 0x01101222, 0x01111021, 0x01111121,- 0x01111122, 0x01111220, 0x01111221, 0x01121021, 0x01121120, 0x01121121, 0x01121220, 0x01121221, 0x01121222,- 0x02101122, 0x02101222, 0x02111022, 0x02111121, 0x02121120, 0x02121221, 0x00112001, 0x00112102, 0x00122101,- 0x01102001, 0x01102100, 0x01102102, 0x01102201, 0x01112000, 0x01112101, 0x01112200, 0x01112202, 0x01122000,- 0x01122001, 0x01122100, 0x01122102, 0x01122201, 0x02102101, 0x02112001, 0x02112100, 0x02122101, 0x00112010,- 0x00112012, 0x00112111, 0x00112212, 0x00122011, 0x00122111, 0x01102012, 0x01102110, 0x01102111, 0x01102210,- 0x01112011, 0x01112110, 0x01112111, 0x01112112, 0x01112211, 0x01112212, 0x01122010, 0x01122111, 0x01122212,- 0x02102211, 0x02112011, 0x02112012, 0x02112111, 0x02112210, 0x02122011, 0x02122112, 0x02122211, 0x00102221,- 0x00112122, 0x00122120, 0x00122122, 0x01102120, 0x01102122, 0x01102221, 0x01112020, 0x01112022, 0x01112121,- 0x01112220, 0x01122021, 0x01122122, 0x01122221, 0x02102121, 0x02112021, 0x02112122, 0x02112222, 0x00200000,- 0x00200002, 0x00200200, 0x00200202, 0x00210101, 0x00220000, 0x00220002, 0x00220101, 0x00220200, 0x00220202,- 0x01200101, 0x01210001, 0x01210201, 0x01220001, 0x01220101, 0x02200000, 0x02200002, 0x02200200, 0x02200202,- 0x02210101, 0x02220000, 0x02220002, 0x02220101, 0x02220200, 0x02220202, 0x00200111, 0x00210011, 0x00210110,- 0x00210211, 0x00220111, 0x01200012, 0x01200110, 0x01200211, 0x01210111, 0x01210210, 0x01210212, 0x01220011,- 0x01220110, 0x01220111, 0x01220112, 0x02200111, 0x02210010, 0x02210112, 0x02210211, 0x02220111, 0x00200021,- 0x00200220, 0x00200222, 0x00210021, 0x00210121, 0x00220020, 0x00220022, 0x00220220, 0x00220222, 0x01200121,- 0x01210021, 0x01210122, 0x01210221, 0x01220121, 0x02200021, 0x02200220, 0x02200222, 0x02210021, 0x02210121,- 0x02220020, 0x02220022, 0x02220220, 0x02220222, 0x00201101, 0x00211100, 0x00211102, 0x00211201, 0x00221101,- 0x01201100, 0x01201101, 0x01201102, 0x01201201, 0x01211002, 0x01211101, 0x01211200, 0x01211202, 0x01221102,- 0x02201101, 0x02211001, 0x02211100, 0x02211201, 0x02221001, 0x02221101, 0x00201211, 0x00211111, 0x00221011,- 0x00221211, 0x01201010, 0x01201111, 0x01201210, 0x01211011, 0x01211110, 0x01211111, 0x01211211, 0x01221012,- 0x01221111, 0x01221210, 0x02201211, 0x02211010, 0x02211110, 0x02211111, 0x02211210, 0x02211212, 0x02221011,- 0x02221110, 0x02221112, 0x02221211, 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, 0x01201021,- 0x01201221, 0x01211121, 0x01221020, 0x01221021, 0x01221221, 0x02201120, 0x02201122, 0x02211020, 0x02211222,- 0x00202000, 0x00202002, 0x00202200, 0x00202202, 0x00212101, 0x00222000, 0x00222002, 0x00222200, 0x00222202,- 0x01202101, 0x01212001, 0x01212100, 0x01222101, 0x02202000, 0x02202002, 0x02202200, 0x02202202, 0x02222000,- 0x02222002, 0x02222200, 0x02222202, 0x00202211, 0x00212011, 0x00212110, 0x00212211, 0x00222111, 0x01202112,- 0x01202211, 0x01212012, 0x01212111, 0x01222011, 0x01222110, 0x01222112, 0x01222211, 0x02202111, 0x02212010,- 0x02212112, 0x02212211, 0x02222110, 0x02222111, 0x00202020, 0x00202022, 0x00202220, 0x00202222, 0x00222020,- 0x00222022, 0x00222220, 0x00222222, 0x01202121, 0x01212021, 0x01212122, 0x01212221, 0x01222121, 0x02202020,- 0x02202022, 0x02202220, 0x02202222, 0x02212121, 0x02222020, 0x02222022, 0x02222220, 0x02222222, 0x10000101,- 0x10010001, 0x10010102, 0x10020101, 0x11000201, 0x11010002, 0x11010101, 0x11010200, 0x11010202, 0x11020001,- 0x11020100, 0x11020102, 0x12010100, 0x12010201, 0x12020001, 0x12020102, 0x10000010, 0x10000011, 0x10000110,- 0x10000112, 0x10000211, 0x10010012, 0x10010111, 0x10010112, 0x10010210, 0x10010212, 0x10020011, 0x10020112,- 0x10020211, 0x11000111, 0x11000210, 0x11000212, 0x11010011, 0x11010110, 0x11010111, 0x11010112, 0x11010211,- 0x11010212, 0x11020111, 0x11020210, 0x11020212, 0x12000011, 0x12000110, 0x12000112, 0x12010010, 0x12010012,- 0x12010111, 0x12020010, 0x12020011, 0x12020012, 0x10000121, 0x10010021, 0x10010120, 0x10010122, 0x10020121,- 0x11000021, 0x11010022, 0x11010121, 0x11010222, 0x11020120, 0x11020221, 0x12000221, 0x12010120, 0x12020121,- 0x10001001, 0x10011101, 0x10011201, 0x10021201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100,- 0x11011101, 0x11011102, 0x11021001, 0x11021002, 0x11021101, 0x11021200, 0x11021202, 0x12001001, 0x12001102,- 0x12001201, 0x12011000, 0x12011002, 0x12011101, 0x12021000, 0x12021001, 0x12021201, 0x10001011, 0x10001012,- 0x10001111, 0x10001212, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010, 0x10021111,- 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010, 0x11011011, 0x11011110,- 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110, 0x11021111, 0x11021112, 0x11021211,- 0x12001012, 0x12001110, 0x12001111, 0x12001210, 0x12011011, 0x12011110, 0x12011111, 0x12011112, 0x12011211,- 0x12011212, 0x12021111, 0x12021210, 0x12021212, 0x10001021, 0x10001121, 0x10001221, 0x10011120, 0x10011121,- 0x10011220, 0x10011222, 0x10021021, 0x10021120, 0x10021221, 0x11001020, 0x11001022, 0x11001121, 0x11001220,- 0x11011020, 0x11011021, 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021022, 0x11021121, 0x11021220,- 0x12001021, 0x12001121, 0x12001222, 0x12011120, 0x12011121, 0x12021021, 0x12021120, 0x12021122, 0x10002101,- 0x10012001, 0x10012101, 0x10012202, 0x10022101, 0x11002002, 0x11002201, 0x11012000, 0x11012101, 0x11012200,- 0x11022001, 0x11022100, 0x11022102, 0x11022201, 0x12002101, 0x12012001, 0x12012100, 0x12012102, 0x12012201,- 0x12022101, 0x10002011, 0x10002111, 0x10002112, 0x10002212, 0x10012010, 0x10012110, 0x10012111, 0x10012210,- 0x10022011, 0x10022110, 0x10022112, 0x11002010, 0x11002111, 0x11002212, 0x11012011, 0x11012012, 0x11012110,- 0x11012111, 0x11012112, 0x11012211, 0x11022010, 0x11022012, 0x11022111, 0x11022112, 0x11022212, 0x12002112,- 0x12002211, 0x12012012, 0x12012111, 0x12012112, 0x12012210, 0x12022011, 0x12022110, 0x12022112, 0x12022211,- 0x10012122, 0x11002120, 0x11002122, 0x11002221, 0x11012121, 0x11012220, 0x11012222, 0x11022120, 0x11022221,- 0x12012120, 0x12022121, 0x10100001, 0x10100100, 0x10100101, 0x10100102, 0x10100201, 0x10110002, 0x10110101,- 0x10110202, 0x10120001, 0x10120100, 0x10120201, 0x11100000, 0x11100101, 0x11100200, 0x11110001, 0x11110100,- 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x11120200, 0x12100102, 0x12100201, 0x12110101, 0x12110200,- 0x12120000, 0x12120001, 0x12120102, 0x12120201, 0x10100111, 0x10100210, 0x10100211, 0x10100212, 0x10110011,- 0x10110110, 0x10110111, 0x10110112, 0x10110210, 0x10110211, 0x10120010, 0x10120111, 0x10120112, 0x10120210,- 0x10120212, 0x11100011, 0x11100110, 0x11100111, 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012,- 0x11110110, 0x11110111, 0x11110112, 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111,- 0x11120112, 0x11120211, 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211,- 0x12120010, 0x12120111, 0x12120212, 0x10100021, 0x10100122, 0x10110022, 0x10110121, 0x10110222, 0x10120021,- 0x10120120, 0x11100022, 0x11100121, 0x11100222, 0x11110021, 0x11110120, 0x11110121, 0x11110122, 0x11110221,- 0x11120022, 0x11120121, 0x12100121, 0x12110020, 0x12110022, 0x12110121, 0x12110221, 0x12110222, 0x12120120,- 0x10101100, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102, 0x10111200, 0x10111201, 0x10121001,- 0x10121101, 0x10121200, 0x10121202, 0x11101001, 0x11101100, 0x11101101, 0x11101102, 0x11101201, 0x11101202,- 0x11111000, 0x11111001, 0x11111100, 0x11111101, 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001,- 0x11121002, 0x11121100, 0x11121101, 0x11121102, 0x11121201, 0x12101000, 0x12101200, 0x12101202, 0x12111001,- 0x12111100, 0x12111101, 0x12111102, 0x12111201, 0x12121001, 0x12121100, 0x12121101, 0x12121202, 0x10101011,- 0x10101012, 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110,- 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112, 0x10121211,- 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210, 0x11101211, 0x11111010,- 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210, 0x11111211, 0x11111212, 0x11121010,- 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210, 0x11121211, 0x11121212, 0x12101011, 0x12101110,- 0x12101111, 0x12101211, 0x12101212, 0x12111010, 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210,- 0x12111211, 0x12121011, 0x12121110, 0x12121111, 0x12121112, 0x12121211, 0x10101020, 0x10101021, 0x10101022,- 0x10101120, 0x10101122, 0x10101220, 0x10101221, 0x10111021, 0x10111120, 0x10111121, 0x10111220, 0x10111221,- 0x10121020, 0x10121021, 0x10121022, 0x10121120, 0x10121121, 0x10121122, 0x10121220, 0x10121221, 0x11101021,- 0x11101121, 0x11101122, 0x11101220, 0x11101221, 0x11101222, 0x11111020, 0x11111021, 0x11111022, 0x11111120,- 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120, 0x11121121, 0x11121221,- 0x12101022, 0x12101121, 0x12101122, 0x12101220, 0x12101221, 0x12101222, 0x12111021, 0x12111121, 0x12111222,- 0x12121022, 0x12121121, 0x12121122, 0x12121220, 0x12121221, 0x10102100, 0x10102101, 0x10102102, 0x10102201,- 0x10112000, 0x10112101, 0x10112200, 0x10122001, 0x10122202, 0x11102101, 0x11102200, 0x11102202, 0x11112001,- 0x11112100, 0x11112101, 0x11112102, 0x11112200, 0x11112201, 0x11122000, 0x11122002, 0x11122100, 0x11122101,- 0x12102002, 0x12102201, 0x12112000, 0x12112002, 0x12112101, 0x12112200, 0x12122001, 0x12122201, 0x10102011,- 0x10102012, 0x10102111, 0x10102212, 0x10112011, 0x10112110, 0x10112111, 0x10112112, 0x10112211, 0x10122111,- 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012, 0x11112110,- 0x11112111, 0x11112112, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110, 0x11122111, 0x11122112,- 0x11122211, 0x12102011, 0x12102111, 0x12102211, 0x12112011, 0x12112110, 0x12112111, 0x12112112, 0x12112210,- 0x12112211, 0x12122111, 0x10102120, 0x10102220, 0x10112121, 0x10112222, 0x10122020, 0x10122121, 0x10122122,- 0x10122221, 0x11102121, 0x11102220, 0x11102221, 0x11112021, 0x11112121, 0x11112122, 0x11112220, 0x11112221,- 0x11122022, 0x11122121, 0x11122220, 0x11122222, 0x12102021, 0x12102222, 0x12112022, 0x12112121, 0x12112122,- 0x12112220, 0x12112222, 0x12122021, 0x10200101, 0x10210100, 0x10210102, 0x10210201, 0x10220101, 0x11200100,- 0x11210000, 0x11210101, 0x11210102, 0x11210200, 0x11210202, 0x11220001, 0x11220100, 0x11220102, 0x11220201,- 0x12200001, 0x12210102, 0x12220101, 0x10200011, 0x10200110, 0x10200112, 0x10200211, 0x10210012, 0x10210111,- 0x10220011, 0x10220012, 0x10220112, 0x10220211, 0x11200111, 0x11200211, 0x11210011, 0x11210111, 0x11210112,- 0x11210211, 0x11220111, 0x11220112, 0x11220212, 0x12200110, 0x12200212, 0x12210012, 0x12210111, 0x12220011,- 0x12220112, 0x12220211, 0x10210021, 0x10210122, 0x10210221, 0x11200020, 0x11200021, 0x11200122, 0x11210121,- 0x11210122, 0x11210220, 0x11220020, 0x12200121, 0x12210021, 0x12210122, 0x12220121, 0x10211001, 0x10211002,- 0x10211101, 0x10211102, 0x10211202, 0x10221001, 0x10221102, 0x10221201, 0x11201000, 0x11201002, 0x11201101,- 0x11201200, 0x11201202, 0x11211001, 0x11211100, 0x11211101, 0x11211102, 0x11211201, 0x11211202, 0x11221000,- 0x11221002, 0x11221101, 0x12201100, 0x12201101, 0x12201201, 0x12211000, 0x12211002, 0x12211100, 0x12211101,- 0x12211102, 0x12211200, 0x12211202, 0x12221001, 0x12221100, 0x12221201, 0x10201111, 0x10201210, 0x10201212,- 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201111, 0x11201112, 0x11201211, 0x11211010,- 0x11211011, 0x11211110, 0x11211111, 0x11211112, 0x11211211, 0x11221011, 0x11221110, 0x11221111, 0x11221112,- 0x11221211, 0x12201112, 0x12201211, 0x12201212, 0x12211011, 0x12211111, 0x12211112, 0x12211211, 0x12211212,- 0x12221012, 0x12221111, 0x12221112, 0x12221210, 0x10201022, 0x10201221, 0x10211121, 0x10221020, 0x10221122,- 0x10221220, 0x10221221, 0x11201020, 0x11201121, 0x11201220, 0x11201222, 0x11211021, 0x11211120, 0x11211121,- 0x11211122, 0x11211220, 0x11211222, 0x11221020, 0x11221121, 0x11221220, 0x12201020, 0x12201022, 0x12201121,- 0x12201222, 0x12211120, 0x12211122, 0x12211220, 0x12211221, 0x12221020, 0x12221120, 0x12221122, 0x12221222,- 0x10212102, 0x10212201, 0x10222101, 0x11202001, 0x11212002, 0x11212101, 0x11212202, 0x11222001, 0x11222201,- 0x12202101, 0x12212001, 0x12212200, 0x12222102, 0x10202011, 0x10202110, 0x10212010, 0x10212111, 0x10222011,- 0x10222110, 0x10222112, 0x10222211, 0x11202010, 0x11202011, 0x11202111, 0x11202112, 0x11202210, 0x11212011,- 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222010, 0x11222111, 0x11222212, 0x12202012, 0x12202110,- 0x12202212, 0x12212111, 0x12222011, 0x12222110, 0x12222111, 0x12222211, 0x10212021, 0x10212122, 0x10212220,- 0x11202021, 0x11202120, 0x11202221, 0x11212020, 0x11212121, 0x11212220, 0x11212222, 0x11222120, 0x11222121,- 0x11222221, 0x12202122, 0x12212120, 0x12212220, 0x12212222, 0x12222122, 0x20000000, 0x20000002, 0x20000200,- 0x20000202, 0x20020000, 0x20020002, 0x20020200, 0x20020202, 0x21000101, 0x21010000, 0x21010001, 0x21010100,- 0x21010102, 0x21010201, 0x21020101, 0x22000000, 0x22000002, 0x22000200, 0x22000202, 0x22010101, 0x22020000,- 0x22020002, 0x22020200, 0x22020202, 0x20000111, 0x20010011, 0x20010110, 0x20010112, 0x20010211, 0x20020111,- 0x21000011, 0x21000110, 0x21000211, 0x21010010, 0x21010012, 0x21010111, 0x21010112, 0x21010210, 0x21010211,- 0x21020110, 0x21020112, 0x21020211, 0x22000111, 0x22000211, 0x22010110, 0x22010112, 0x22010211, 0x22020111,- 0x20000020, 0x20000022, 0x20000220, 0x20000222, 0x20010121, 0x20020020, 0x20020022, 0x20020220, 0x20020222,- 0x21010021, 0x21010120, 0x21010221, 0x21020121, 0x22000020, 0x22000022, 0x22000220, 0x22000222, 0x22010121,- 0x22020020, 0x22020022, 0x22020220, 0x22020222, 0x20011100, 0x20011201, 0x21001001, 0x21001100, 0x21011001,- 0x21011101, 0x21011202, 0x21021001, 0x21021100, 0x21021201, 0x22011100, 0x22011201, 0x20001011, 0x20001211,- 0x20011012, 0x20011111, 0x20011212, 0x20021112, 0x20021211, 0x21001010, 0x21001011, 0x21001111, 0x21001210,- 0x21011011, 0x21011110, 0x21011111, 0x21011112, 0x21011211, 0x21011212, 0x21021111, 0x21021112, 0x21021210,- 0x21021212, 0x22001011, 0x22001110, 0x22001112, 0x22001211, 0x22011010, 0x22011012, 0x22011111, 0x22011210,- 0x22021112, 0x20011021, 0x20011122, 0x20011221, 0x20021121, 0x21001021, 0x21001120, 0x21001221, 0x21001222,- 0x21011020, 0x21011121, 0x21011221, 0x21011222, 0x21021021, 0x21021122, 0x21021222, 0x22001121, 0x22011021,- 0x22011222, 0x22021120, 0x20002000, 0x20002002, 0x20002200, 0x20002202, 0x20012101, 0x20022000, 0x20022002,- 0x20022200, 0x20022202, 0x21002001, 0x21002101, 0x21012001, 0x21012100, 0x21012201, 0x21022101, 0x21022201,- 0x22002000, 0x22002002, 0x22002200, 0x22002202, 0x22012101, 0x22022000, 0x22022002, 0x22022200, 0x22022202,- 0x20002111, 0x20002112, 0x20012011, 0x20012110, 0x20012112, 0x20022111, 0x21002011, 0x21002110, 0x21002112,- 0x21002211, 0x21012010, 0x21012012, 0x21012111, 0x21012212, 0x21022011, 0x21022110, 0x22002111, 0x22012112,- 0x22012211, 0x22022111, 0x20002020, 0x20002022, 0x20002220, 0x20002222, 0x20012121, 0x20022020, 0x20022022,- 0x20022220, 0x20022222, 0x21002121, 0x21012021, 0x21012120, 0x21012122, 0x22002020, 0x22002022, 0x22002220,- 0x22002222, 0x22012121, 0x22022020, 0x22022022, 0x22022220, 0x22022222, 0x20100101, 0x20110001, 0x20110102,- 0x20110200, 0x20110201, 0x20120101, 0x21100001, 0x21100102, 0x21100201, 0x21110101, 0x21110200, 0x21110202,- 0x21120201, 0x21120202, 0x22100101, 0x22110001, 0x22110100, 0x22110102, 0x22110201, 0x22120101, 0x20100011,- 0x20100110, 0x20100112, 0x20100211, 0x20110010, 0x20110111, 0x20110210, 0x20110212, 0x20120011, 0x20120110,- 0x20120112, 0x20120211, 0x21100010, 0x21100111, 0x21110010, 0x21110011, 0x21110110, 0x21110111, 0x21110112,- 0x21110211, 0x21120012, 0x21120111, 0x22100110, 0x22100112, 0x22110012, 0x22110111, 0x22110210, 0x22120011,- 0x22120110, 0x22120112, 0x22120211, 0x20100121, 0x20110021, 0x20110120, 0x20110221, 0x20120121, 0x21100120,- 0x21100122, 0x21100221, 0x21110020, 0x21110022, 0x21110121, 0x21110220, 0x21120122, 0x21120221, 0x22100121,- 0x22110120, 0x22110122, 0x22120221, 0x20101001, 0x20101100, 0x20101102, 0x20111000, 0x20111101, 0x20111200,- 0x20121102, 0x21101000, 0x21101202, 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, 0x21111201,- 0x21121000, 0x21121001, 0x21121002, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111100, 0x22111101,- 0x22111200, 0x22121001, 0x22121201, 0x20101010, 0x20101111, 0x20101210, 0x20101212, 0x20111010, 0x20111011,- 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, 0x20121111, 0x20121211, 0x20121212, 0x21101011,- 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111012, 0x21111110, 0x21111111,- 0x21111112, 0x21111210, 0x21111211, 0x21111212, 0x21121011, 0x21121110, 0x21121111, 0x21121112, 0x21121211,- 0x22101011, 0x22101111, 0x22101210, 0x22111011, 0x22111012, 0x22111110, 0x22111111, 0x22111112, 0x22111211,- 0x22111212, 0x22121010, 0x22121012, 0x22121111, 0x22121210, 0x22121212, 0x20101021, 0x20101120, 0x20111020,- 0x20111121, 0x20111221, 0x20121020, 0x20121122, 0x20121221, 0x21101121, 0x21101220, 0x21101221, 0x21111021,- 0x21111022, 0x21111121, 0x21111122, 0x21111221, 0x21121121, 0x21121220, 0x22101022, 0x22101120, 0x22101221,- 0x22101222, 0x22111022, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x22121221, 0x20102101, 0x20112102,- 0x20112201, 0x20122101, 0x21102001, 0x21102102, 0x21112000, 0x21112002, 0x21112101, 0x21112102, 0x21112202,- 0x21122100, 0x21122101, 0x22102101, 0x22112001, 0x22112102, 0x22112201, 0x22122101, 0x20102110, 0x20102112,- 0x20102211, 0x20112010, 0x20112012, 0x20112111, 0x20112210, 0x20112212, 0x20122010, 0x20122011, 0x20122110,- 0x20122112, 0x21102010, 0x21102012, 0x21102111, 0x21102210, 0x21102212, 0x21112011, 0x21112110, 0x21112111,- 0x21112112, 0x21112211, 0x21122012, 0x21122111, 0x21122112, 0x21122212, 0x22102011, 0x22102110, 0x22112010,- 0x22112012, 0x22112111, 0x22112212, 0x22122011, 0x22122112, 0x20102121, 0x20112121, 0x20122121, 0x21102120,- 0x21102122, 0x21102221, 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22102121, 0x22112021, 0x22112120,- 0x22112121, 0x22112122, 0x20200000, 0x20200002, 0x20200200, 0x20200202, 0x20210101, 0x20220000, 0x20220002,- 0x20220200, 0x20220202, 0x21200101, 0x21210001, 0x21210100, 0x21210102, 0x21210201, 0x22200000, 0x22200002,- 0x22200200, 0x22200202, 0x22210101, 0x22220000, 0x22220002, 0x22220200, 0x22220202, 0x20200111, 0x20200211,- 0x20210011, 0x20210110, 0x20210112, 0x20210211, 0x20210212, 0x21200112, 0x21200211, 0x21210011, 0x21210111,- 0x21210210, 0x21210212, 0x21220011, 0x21220110, 0x22200111, 0x22210010, 0x22210012, 0x22210112, 0x22210211,- 0x20200022, 0x20200220, 0x20200222, 0x20210020, 0x20210221, 0x20220022, 0x20220220, 0x20220222, 0x21200121,- 0x21210021, 0x21210122, 0x21210221, 0x21220121, 0x22200020, 0x22200022, 0x22200220, 0x22200222, 0x22210121,- 0x22220020, 0x22220022, 0x22220220, 0x22220222, 0x20211201, 0x20221101, 0x21201001, 0x21201100, 0x21211000,- 0x21211100, 0x21211101, 0x21211200, 0x21211202, 0x21221001, 0x21221101, 0x21221102, 0x21221200, 0x21221201,- 0x22201101, 0x20201112, 0x20201211, 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x20221112, 0x20221211,- 0x21201012, 0x21201111, 0x21211011, 0x21211110, 0x21211111, 0x21211112, 0x21211211, 0x21221111, 0x21221212,- 0x22201011, 0x22201110, 0x22201111, 0x22201112, 0x22201211, 0x22211012, 0x22211111, 0x22211210, 0x20201121,- 0x20211021, 0x20211122, 0x20211222, 0x20221021, 0x20221121, 0x21201120, 0x21201122, 0x21201222, 0x21211022,- 0x21211121, 0x21211122, 0x21211220, 0x21221020, 0x21221022, 0x22201122, 0x22211020, 0x22211121, 0x22211122,- 0x22211221, 0x22221021, 0x22221120, 0x22221122, 0x20202000, 0x20202002, 0x20202200, 0x20202202, 0x20222000,- 0x20222002, 0x20222200, 0x20222202, 0x21212001, 0x21212100, 0x21212102, 0x21212201, 0x22202000, 0x22202002,- 0x22202200, 0x22202202, 0x22212101, 0x22222000, 0x22222002, 0x22222200, 0x22222202, 0x20202111, 0x20212110,- 0x20212211, 0x20222011, 0x20222111, 0x21202011, 0x21212010, 0x21212111, 0x21212212, 0x21222011, 0x21222112,- 0x21222211, 0x22212010, 0x22212112, 0x20202020, 0x20202022, 0x20202220, 0x20202222, 0x20222020, 0x20222022,- 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020, 0x22202022, 0x22202220, 0x22202222,- 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222,-};--static const __device__ uint8_t ksigns_iq2xs[128] = {- 0, 129, 130, 3, 132, 5, 6, 135, 136, 9, 10, 139, 12, 141, 142, 15, 144, 17, 18, 147, 20, 149,- 150, 23, 24, 153, 154, 27, 156, 29, 30, 159, 160, 33, 34, 163, 36, 165, 166, 39, 40, 169, 170, 43,- 172, 45, 46, 175, 48, 177, 178, 51, 180, 53, 54, 183, 184, 57, 58, 187, 60, 189, 190, 63, 192, 65,- 66, 195, 68, 197, 198, 71, 72, 201, 202, 75, 204, 77, 78, 207, 80, 209, 210, 83, 212, 85, 86, 215,- 216, 89, 90, 219, 92, 221, 222, 95, 96, 225, 226, 99, 228, 101, 102, 231, 232, 105, 106, 235, 108, 237,- 238, 111, 240, 113, 114, 243, 116, 245, 246, 119, 120, 249, 250, 123, 252, 125, 126, 255,-};--static const __device__ uint64_t ksigns64[128] = {- 0x0000000000000000, 0xff000000000000ff, 0xff0000000000ff00, 0x000000000000ffff, 0xff00000000ff0000,- 0x0000000000ff00ff, 0x0000000000ffff00, 0xff00000000ffffff, 0xff000000ff000000, 0x00000000ff0000ff,- 0x00000000ff00ff00, 0xff000000ff00ffff, 0x00000000ffff0000, 0xff000000ffff00ff, 0xff000000ffffff00,- 0x00000000ffffffff, 0xff0000ff00000000, 0x000000ff000000ff, 0x000000ff0000ff00, 0xff0000ff0000ffff,- 0x000000ff00ff0000, 0xff0000ff00ff00ff, 0xff0000ff00ffff00, 0x000000ff00ffffff, 0x000000ffff000000,- 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0x000000ffff00ffff, 0xff0000ffffff0000, 0x000000ffffff00ff,- 0x000000ffffffff00, 0xff0000ffffffffff, 0xff00ff0000000000, 0x0000ff00000000ff, 0x0000ff000000ff00,- 0xff00ff000000ffff, 0x0000ff0000ff0000, 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0x0000ff0000ffffff,- 0x0000ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00, 0x0000ff00ff00ffff, 0xff00ff00ffff0000,- 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0xff00ff00ffffffff, 0x0000ffff00000000, 0xff00ffff000000ff,- 0xff00ffff0000ff00, 0x0000ffff0000ffff, 0xff00ffff00ff0000, 0x0000ffff00ff00ff, 0x0000ffff00ffff00,- 0xff00ffff00ffffff, 0xff00ffffff000000, 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0xff00ffffff00ffff,- 0x0000ffffffff0000, 0xff00ffffffff00ff, 0xff00ffffffffff00, 0x0000ffffffffffff, 0xffff000000000000,- 0x00ff0000000000ff, 0x00ff00000000ff00, 0xffff00000000ffff, 0x00ff000000ff0000, 0xffff000000ff00ff,- 0xffff000000ffff00, 0x00ff000000ffffff, 0x00ff0000ff000000, 0xffff0000ff0000ff, 0xffff0000ff00ff00,- 0x00ff0000ff00ffff, 0xffff0000ffff0000, 0x00ff0000ffff00ff, 0x00ff0000ffffff00, 0xffff0000ffffffff,- 0x00ff00ff00000000, 0xffff00ff000000ff, 0xffff00ff0000ff00, 0x00ff00ff0000ffff, 0xffff00ff00ff0000,- 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00, 0xffff00ff00ffffff, 0xffff00ffff000000, 0x00ff00ffff0000ff,- 0x00ff00ffff00ff00, 0xffff00ffff00ffff, 0x00ff00ffffff0000, 0xffff00ffffff00ff, 0xffff00ffffffff00,- 0x00ff00ffffffffff, 0x00ffff0000000000, 0xffffff00000000ff, 0xffffff000000ff00, 0x00ffff000000ffff,- 0xffffff0000ff0000, 0x00ffff0000ff00ff, 0x00ffff0000ffff00, 0xffffff0000ffffff, 0xffffff00ff000000,- 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0xffffff00ff00ffff, 0x00ffff00ffff0000, 0xffffff00ffff00ff,- 0xffffff00ffffff00, 0x00ffff00ffffffff, 0xffffffff00000000, 0x00ffffff000000ff, 0x00ffffff0000ff00,- 0xffffffff0000ffff, 0x00ffffff00ff0000, 0xffffffff00ff00ff, 0xffffffff00ffff00, 0x00ffffff00ffffff,- 0x00ffffffff000000, 0xffffffffff0000ff, 0xffffffffff00ff00, 0x00ffffffff00ffff, 0xffffffffffff0000,- 0x00ffffffffff00ff, 0x00ffffffffffff00, 0xffffffffffffffff,-};--static const __device__ uint8_t kmask_iq2xs[8] = {1, 2, 4, 8, 16, 32, 64, 128};-static const __device__ int8_t kvalues_iq4nl[16] = {- -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113};--typedef half dfloat; // dequantize float-typedef half2 dfloat2;-typedef void (*dequantize_kernel_t)(const void* vx, const int ib, const int iqs, dfloat2& v);-template <typename dst_t>-using to_cuda_ggml_t = void (*)(const void* __restrict__ x, dst_t* __restrict__ y, int k, cudaStream_t stream);-typedef float (*vec_dot_q_cuda_t)(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs);-typedef void (*allocate_tiles_cuda_t)(int** x_ql, half2** x_dm, int** x_qh, int** x_sc);-typedef void (*load_tiles_cuda_t)(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row);-typedef float (*vec_dot_q_mul_mat_cuda_t)(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ms,- const int& i,- const int& j,- const int& k);--// Utility function--template <typename dst_t>-static __device__ __forceinline__ dst_t convert_from_half(half val) {- return val;-}--template <>-__device__ __forceinline__ c10::BFloat16 convert_from_half<c10::BFloat16>(half val) {-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA)- return __float2bfloat16(__half2float(val));-#else- return __half2float(val);-#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA)-}--template <>-__device__ __forceinline__ float convert_from_half<float>(half val) {- return __half2float(val);-}--#if defined(USE_ROCM)--#ifndef __has_builtin-#define __has_builtin(x) 0-#endif--typedef int8_t int8x4_t __attribute__((ext_vector_type(4)));-static __device__ __forceinline__ int __vsubss4(const int a, const int b) {- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);-#if __has_builtin(__builtin_elementwise_sub_sat)- const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);- return reinterpret_cast<const int&>(c);-#else- int8x4_t c;- int16_t tmp;-#pragma unroll- for (int i = 0; i < 4; i++) {- tmp = va[i] - vb[i];- if (tmp > std::numeric_limits<int8_t>::max()) tmp = std::numeric_limits<int8_t>::max();- if (tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();- c[i] = tmp;- }- return reinterpret_cast<int&>(c);-#endif // __has_builtin(__builtin_elementwise_sub_sat)-}--static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {-#if __has_builtin(__builtin_amdgcn_sdot4)- c = __builtin_amdgcn_sdot4(a, b, c, false);-#else- const int8x4_t va = reinterpret_cast<const int8x4_t&>(a);- const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);- c += va[0] * vb[0] + va[1] * vb[1] + va[2] * vb[2] + va[3] * vb[3];-#endif- return c;-}--static __device__ __forceinline__ uint32_t __vcmpeq4(const uint32_t a, const uint32_t b) {- uint32_t neq = a ^ b;- return !(neq & 0xff000000) * 0xff000000 | !(neq & 0x00ff0000) * 0x00ff0000 | !(neq & 0x0000ff00) * 0x0000ff00 |- !(neq & 0x000000ff) * 0x000000ff;-}--static __device__ __forceinline__ uint32_t __vsub4(const uint32_t a, const uint32_t b) {- return (static_cast<uint8_t>(((a & 0xff000000) >> 24) - ((b & 0xff000000) >> 24)) << 24) +- (static_cast<uint8_t>(((a & 0x00ff0000) >> 16) - ((b & 0x00ff0000) >> 16)) << 16) +- (static_cast<uint8_t>(((a & 0x0000ff00) >> 8) - ((b & 0x0000ff00) >> 8)) << 8) +- (static_cast<uint8_t>(((a & 0x000000ff) >> 0) - ((b & 0x000000ff) >> 0)) << 0);-}-#endif // defined(USE_ROCM)diff --git a/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu b/python/freetoken/kernel/csrc/gguf/gguf_kernel.cudeleted file mode 100644index d88960d5..00000000--- a/python/freetoken/kernel/csrc/gguf/gguf_kernel.cu+++ /dev/null@@ -1,848 +0,0 @@-// Adatped from-// https://github.com/vllm-project/vllm/blob/755ed7b05be4743237d3339c4ff8c22bcaae04f4/csrc/quantization/gguf/gguf_kernel.cu-#include <c10/cuda/CUDAGuard.h>-#include <cuda_fp16.h>-#include <cuda_runtime.h>-#include <torch/all.h>--// dont use clang-format here, it breaks the include order-// clang-format off-#include "dispatch.h"--#include "ggml-common.h"-#include "vecdotq.cuh"-#include "dequantize.cuh"-#include "mmvq.cuh"-#include "mmq.cuh"-#include "moe.cuh"-#include "moe_vec.cuh"-// clang-format off--// Q8 gemv-template <typename scalar_t>-static __global__ void-quantize_q8_1(const scalar_t* __restrict__ x, void* __restrict__ vy, const int kx, const int kx_padded) {- const auto ix = blockDim.x * blockIdx.x + threadIdx.x;- if (ix >= kx_padded) {- return;- }- const auto iy = blockDim.y * blockIdx.y + threadIdx.y;- const int i_padded = iy * kx_padded + ix;-- block_q8_1* y = (block_q8_1*)vy;-- const int ib = i_padded / QK8_1; // block index- const int iqs = i_padded % QK8_1; // quant index-- const float xi = ix < kx ? static_cast<float>(x[iy * kx + ix]) : 0.0f;- float amax = fabsf(xi);- float sum = xi;--#pragma unroll- for (int mask = 16; mask > 0; mask >>= 1) {- amax = fmaxf(amax, SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), amax, mask, 32));- sum += SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), sum, mask, 32);- }-- const float d = amax / 127;- const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);-- y[ib].qs[iqs] = q;-- if (iqs > 0) {- return;- }-- y[ib].ds.x = __float2half(d);- y[ib].ds.y = __float2half(sum);-}--template <typename scalar_t>-static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx, const int ky, cudaStream_t stream) {- const int64_t kx_padded = (kx + 512 - 1) / 512 * 512;- const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;- constexpr int MAX_BLOCK_SIZE = 65535;- for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) {- const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off;- const dim3 num_blocks(block_num_x, num_blocks_y, 1);- const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);- quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(- &x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded);- }-}--torch::Tensor ggml_dequantize(- torch::Tensor W, // quant weight- int64_t type,- int64_t m,- int64_t n,- std::optional<at::ScalarType> const& dtype) {- const at::cuda::OptionalCUDAGuard device_guard(device_of(W));- auto dtype_ = dtype.value_or(torch::kFloat16);- auto options = torch::TensorOptions().dtype(dtype_).device(W.device());- at::Tensor DW = torch::empty({m, n}, options);- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();-- DISPATCH_FLOAT_TYPES(DW.scalar_type(), "ggml_dequantize", [&] {- auto to_cuda = ggml_get_to_cuda<scalar_t>(type);- to_cuda((void*)W.data_ptr(), (scalar_t*)DW.data_ptr(), m * n, stream);- });-- return DW;-}--torch::Tensor ggml_mul_mat_vec_a8(- torch::Tensor W, // quant weight- torch::Tensor X, // input- int64_t type,- int64_t row) {- int col = X.sizes()[1];- int vecs = X.sizes()[0];- const int padded = (col + 512 - 1) / 512 * 512;- const at::cuda::OptionalCUDAGuard device_guard(device_of(X));- auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device());- at::Tensor Y = torch::empty({vecs, row}, options);- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();- options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());- at::Tensor quant_X = torch::empty({vecs, padded / 32 * 9}, options);- DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_mul_mat_vec_a8", [&] {- quantize_row_q8_1_cuda<scalar_t>((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, vecs, stream);- switch (type) {- case 2:- mul_mat_vec_q4_0_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 3:- mul_mat_vec_q4_1_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 6:- mul_mat_vec_q5_0_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 7:- mul_mat_vec_q5_1_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 8:- mul_mat_vec_q8_0_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 10:- mul_mat_vec_q2_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 11:- mul_mat_vec_q3_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 12:- mul_mat_vec_q4_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 13:- mul_mat_vec_q5_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 14:- mul_mat_vec_q6_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 16:- mul_mat_vec_iq2_xxs_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 17:- mul_mat_vec_iq2_xs_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 18:- mul_mat_vec_iq3_xxs_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 19:- mul_mat_vec_iq1_s_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 20:- mul_mat_vec_iq4_nl_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 21:- mul_mat_vec_iq3_s_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 22:- mul_mat_vec_iq2_s_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 23:- mul_mat_vec_iq4_xs_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- case 29:- mul_mat_vec_iq1_m_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(), (void*)quant_X.data_ptr(), (scalar_t*)Y.data_ptr(), col, row, vecs, stream);- break;- }- });- return Y;-}--torch::Tensor ggml_mul_mat_a8(- torch::Tensor W, // quant weight- torch::Tensor X, // input- int64_t type,- int64_t row) {- int col = X.sizes()[1];- int padded = (col + 512 - 1) / 512 * 512;- int batch = X.sizes()[0];- const at::cuda::OptionalCUDAGuard device_guard(device_of(X));- auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device());- at::Tensor Y = torch::empty({batch, row}, options);- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();- options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());- at::Tensor quant_X = torch::empty({batch, padded / 32 * 9}, options);- DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_mul_mat_a8", [&] {- quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, batch, stream);-- switch (type) {- case 2:- ggml_mul_mat_q4_0_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 3:- ggml_mul_mat_q4_1_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 6:- ggml_mul_mat_q5_0_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 7:- ggml_mul_mat_q5_1_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 8:- ggml_mul_mat_q8_0_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 10:- ggml_mul_mat_q2_K_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 11:- ggml_mul_mat_q3_K_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 12:- ggml_mul_mat_q4_K_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 13:- ggml_mul_mat_q5_K_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- case 14:- ggml_mul_mat_q6_K_q8_1_cuda(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- col,- row,- batch,- padded,- row,- stream);- break;- }- });- return Y;-}--torch::Tensor ggml_moe_a8(- torch::Tensor X, // input- torch::Tensor W, // expert weights- torch::Tensor sorted_token_ids,- torch::Tensor expert_ids,- torch::Tensor num_tokens_post_padded,- int64_t type,- int64_t row,- int64_t top_k,- int64_t tokens) {- int col = X.sizes()[1];- int padded = (col + 512 - 1) / 512 * 512;- const at::cuda::OptionalCUDAGuard device_guard(device_of(X));- auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device());- at::Tensor Y = torch::empty({tokens * top_k, row}, options);- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();- options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());- at::Tensor quant_X = torch::empty({tokens, padded / 32 * 9}, options);- DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_moe_a8", [&] {- quantize_row_q8_1_cuda((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, tokens, stream);- switch (type) {- case 2:- ggml_moe_q4_0_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 3:- ggml_moe_q4_1_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 6:- ggml_moe_q5_0_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 7:- ggml_moe_q5_1_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 8:- ggml_moe_q8_0_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 10:- ggml_moe_q2_K_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 11:- ggml_moe_q3_K_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 12:- ggml_moe_q4_K_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 13:- ggml_moe_q5_K_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- case 14:- ggml_moe_q6_K_q8_1_cuda(- (void*)quant_X.data_ptr(),- (void*)W.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)sorted_token_ids.data_ptr(),- (int*)expert_ids.data_ptr(),- (int*)num_tokens_post_padded.data_ptr(),- W.stride(0),- col,- row,- tokens,- padded,- row,- top_k,- sorted_token_ids.sizes()[0],- stream);- break;- }- });- return Y;-}--torch::Tensor ggml_moe_a8_vec(- torch::Tensor X, // input- torch::Tensor W, // expert weights- torch::Tensor topk_ids,- int64_t top_k,- int64_t type,- int64_t row,- int64_t tokens) {- int col = X.sizes()[1];- const int padded = (col + 512 - 1) / 512 * 512;- const at::cuda::OptionalCUDAGuard device_guard(device_of(X));- auto options = torch::TensorOptions().dtype(X.dtype()).device(W.device());- at::Tensor Y = torch::zeros({tokens * top_k, row}, options);- cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream();- options = torch::TensorOptions().dtype(torch::kInt32).device(W.device());- at::Tensor quant_X = torch::empty({tokens, padded / 32 * 9}, options);- DISPATCH_FLOAT_TYPES(X.scalar_type(), "ggml_moe_vec_a8", [&] {- quantize_row_q8_1_cuda<scalar_t>((scalar_t*)X.data_ptr(), (void*)quant_X.data_ptr(), col, tokens, stream);- switch (type) {- case 2:- moe_vec_q4_0_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 3:- moe_vec_q4_1_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 6:- moe_vec_q5_0_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 7:- moe_vec_q5_1_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 8:- moe_vec_q8_0_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 10:- moe_vec_q2_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 11:- moe_vec_q3_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 12:- moe_vec_q4_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 13:- moe_vec_q5_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 14:- moe_vec_q6_K_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 16:- moe_vec_iq2_xxs_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 17:- moe_vec_iq2_xs_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 18:- moe_vec_iq3_xxs_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 19:- moe_vec_iq1_s_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 20:- moe_vec_iq4_nl_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 21:- moe_vec_iq3_s_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 22:- moe_vec_iq2_s_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 23:- moe_vec_iq4_xs_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- case 29:- moe_vec_iq1_m_q8_1_cuda<scalar_t>(- (void*)W.data_ptr(),- (void*)quant_X.data_ptr(),- (scalar_t*)Y.data_ptr(),- (int*)topk_ids.data_ptr(),- top_k,- tokens,- col,- row,- quant_X.stride(0),- stream);- break;- }- });- return Y;-}--int64_t ggml_moe_get_block_size(int64_t type) {- switch (type) {- case 2:- return MOE_X_Q4_0;- case 3:- return MOE_X_Q4_1;- case 6:- return MOE_X_Q5_0;- case 7:- return MOE_X_Q5_1;- case 8:- return MOE_X_Q8_0;- case 10:- return MOE_X_Q2_K;- case 11:- return MOE_X_Q3_K;- case 12:- return MOE_X_Q4_K;- case 13:- return MOE_X_Q5_K;- case 14:- return MOE_X_Q6_K;- }- return 0;-}--// ---- FreeToken pybind bindings (donor registers these via TORCH_LIBRARY; we-// expose them through torch.utils.cpp_extension.load's pybind module instead) -----#include <torch/extension.h>-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {- m.def("ggml_dequantize", &ggml_dequantize, "");- m.def("ggml_mul_mat_vec_a8", &ggml_mul_mat_vec_a8, "");- m.def("ggml_mul_mat_a8", &ggml_mul_mat_a8, "");- m.def("ggml_moe_a8", &ggml_moe_a8, "");- m.def("ggml_moe_a8_vec", &ggml_moe_a8_vec, "");- m.def("ggml_moe_get_block_size", &ggml_moe_get_block_size, "");-}diff --git a/python/freetoken/kernel/csrc/gguf/mmq.cuh b/python/freetoken/kernel/csrc/gguf/mmq.cuhdeleted file mode 100644index 9838c14f..00000000--- a/python/freetoken/kernel/csrc/gguf/mmq.cuh+++ /dev/null@@ -1,881 +0,0 @@-// copied from-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/mmq.cuh-// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu-template <- typename scalar_t,- int qk,- int qr,- int qi,- bool need_sum,- typename block_q_t,- int mmq_x,- int mmq_y,- int nwarps,- allocate_tiles_cuda_t allocate_tiles,- load_tiles_cuda_t load_tiles,- int vdr,- vec_dot_q_mul_mat_cuda_t vec_dot>-static __device__ __forceinline__ void mul_mat_q(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const block_q_t* x = (const block_q_t*)vx;- const block_q8_1* y = (const block_q8_1*)vy;-- const int blocks_per_row_x = ncols_x / qk;- const int blocks_per_col_y = nrows_y / QK8_1;- const int blocks_per_warp = WARP_SIZE_GGUF / qi;-- const int& ncols_dst = ncols_y;-- const auto row_dst_0 = blockIdx.x * mmq_y;- const int& row_x_0 = row_dst_0;-- const auto col_dst_0 = blockIdx.y * mmq_x;- const int& col_y_0 = col_dst_0;-- int* tile_x_ql = nullptr;- half2* tile_x_dm = nullptr;- int* tile_x_qh = nullptr;- int* tile_x_sc = nullptr;-- allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);-- __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];- __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF / QI8_1];-- float sum[mmq_y / WARP_SIZE_GGUF][mmq_x / nwarps] = {{0.0f}};-- for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {- load_tiles(- x + row_x_0 * blocks_per_row_x + ib0,- tile_x_ql,- tile_x_dm,- tile_x_qh,- tile_x_sc,- threadIdx.y,- nrows_x - row_x_0 - 1,- threadIdx.x,- blocks_per_row_x);--#pragma unroll- for (int ir = 0; ir < qr && ib0 + ir * blocks_per_warp / qr < blocks_per_row_x; ++ir) {- const auto kqs = ir * WARP_SIZE_GGUF + threadIdx.x;- const int kbxd = kqs / QI8_1;--#pragma unroll- for (int i = 0; i < mmq_x; i += nwarps) {- const int col_y_eff = min(col_y_0 + threadIdx.y + i, ncols_y - 1); // to prevent out-of-bounds memory accesses- const block_q8_1* by0 = &y[col_y_eff * blocks_per_col_y + ib0 * (qk / QK8_1) + kbxd];- const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;- tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);- }--#pragma unroll- for (int ids0 = 0; ids0 < mmq_x; ids0 += nwarps * QI8_1) {- const int ids = (ids0 + threadIdx.y * QI8_1 + threadIdx.x / (WARP_SIZE_GGUF / QI8_1)) % mmq_x;- const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1);- const int col_y_eff = min(col_y_0 + ids, ncols_y - 1);-- // if the sum is not needed it's faster to transform the scale to f32 ahead of time- const half2* dsi_src =- &y[col_y_eff * blocks_per_col_y + ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby].ds;- half2* dsi_dst = &tile_y_ds[ids * (WARP_SIZE_GGUF / QI8_1) + kby];- if (need_sum) {- *dsi_dst = *dsi_src;- } else {- float* dfi_dst = (float*)dsi_dst;- *dfi_dst = __low2float(*dsi_src);- }- }-- __syncthreads();-- // #pragma unroll // unrolling this loop causes too much register pressure- for (int k = ir * WARP_SIZE_GGUF / qr; k < (ir + 1) * WARP_SIZE_GGUF / qr; k += vdr) {-#pragma unroll- for (int j = 0; j < mmq_x; j += nwarps) {-#pragma unroll- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {- sum[i / WARP_SIZE_GGUF][j / nwarps] += vec_dot(- tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds, threadIdx.x + i, threadIdx.y + j, k);- }- }- }- __syncthreads();- }- }--#pragma unroll- for (int j = 0; j < mmq_x; j += nwarps) {- const auto col_dst = col_dst_0 + j + threadIdx.y;- if (col_dst >= ncols_dst) {- return;- }--#pragma unroll- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {- const auto row_dst = row_dst_0 + threadIdx.x + i;- if (row_dst >= nrows_dst) {- continue;- }- dst[col_dst * nrows_dst + row_dst] = sum[i / WARP_SIZE_GGUF][j / nwarps];- }- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q4_0 64-#define MMQ_Y_Q4_0 128-#define NWARPS_Q4_0 8-#else-#define MMQ_X_Q4_0 4-#define MMQ_Y_Q4_0 32-#define NWARPS_Q4_0 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_0, 2)-#endif- mul_mat_q4_0(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q4_0;- const int mmq_y = MMQ_Y_Q4_0;- const int nwarps = NWARPS_Q4_0;-- mul_mat_q<- scalar_t,- QK4_0,- QR4_0,- QI4_0,- true,- block_q4_0,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q4_0<mmq_y>,- load_tiles_q4_0<mmq_y, nwarps, need_check>,- VDR_Q4_0_Q8_1_MMQ,- vec_dot_q4_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q4_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- int mmq_x = MMQ_X_Q4_0;- int mmq_y = MMQ_Y_Q4_0;- int nwarps = NWARPS_Q4_0;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_0<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_0<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q4_1 64-#define MMQ_Y_Q4_1 128-#define NWARPS_Q4_1 8-#else-#define MMQ_X_Q4_1 4-#define MMQ_Y_Q4_1 32-#define NWARPS_Q4_1 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_1, 2)-#endif- mul_mat_q4_1(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q4_1;- const int mmq_y = MMQ_Y_Q4_1;- const int nwarps = NWARPS_Q4_1;-- mul_mat_q<- scalar_t,- QK4_1,- QR4_1,- QI4_1,- true,- block_q4_1,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q4_1<mmq_y>,- load_tiles_q4_1<mmq_y, nwarps, need_check>,- VDR_Q4_1_Q8_1_MMQ,- vec_dot_q4_1_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q4_1_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- int mmq_x = MMQ_X_Q4_1;- int mmq_y = MMQ_Y_Q4_1;- int nwarps = NWARPS_Q4_1;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_1<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_1<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q5_0 64-#define MMQ_Y_Q5_0 128-#define NWARPS_Q5_0 8-#else-#define MMQ_X_Q5_0 4-#define MMQ_Y_Q5_0 32-#define NWARPS_Q5_0 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_0, 2)-#endif- mul_mat_q5_0(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q5_0;- const int mmq_y = MMQ_Y_Q5_0;- const int nwarps = NWARPS_Q5_0;-- mul_mat_q<- scalar_t,- QK5_0,- QR5_0,- QI5_0,- false,- block_q5_0,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q5_0<mmq_y>,- load_tiles_q5_0<mmq_y, nwarps, need_check>,- VDR_Q5_0_Q8_1_MMQ,- vec_dot_q5_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q5_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- const int mmq_x = MMQ_X_Q5_0;- const int mmq_y = MMQ_Y_Q5_0;- const int nwarps = NWARPS_Q5_0;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_0<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_0<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q5_1 64-#define MMQ_Y_Q5_1 128-#define NWARPS_Q5_1 8-#else-#define MMQ_X_Q5_1 4-#define MMQ_Y_Q5_1 32-#define NWARPS_Q5_1 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_1, 2)-#endif- mul_mat_q5_1(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q5_1;- const int mmq_y = MMQ_Y_Q5_1;- const int nwarps = NWARPS_Q5_1;-- mul_mat_q<- scalar_t,- QK5_1,- QR5_1,- QI5_1,- true,- block_q5_1,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q5_1<mmq_y>,- load_tiles_q5_1<mmq_y, nwarps, need_check>,- VDR_Q5_1_Q8_1_MMQ,- vec_dot_q5_1_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q5_1_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- const int mmq_x = MMQ_X_Q5_1;- const int mmq_y = MMQ_Y_Q5_1;- const int nwarps = NWARPS_Q5_1;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_1<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_1<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q8_0 64-#define MMQ_Y_Q8_0 128-#define NWARPS_Q8_0 8-#else-#define MMQ_X_Q8_0 4-#define MMQ_Y_Q8_0 32-#define NWARPS_Q8_0 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q8_0, 2)-#endif- mul_mat_q8_0(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q8_0;- const int mmq_y = MMQ_Y_Q8_0;- const int nwarps = NWARPS_Q8_0;-- mul_mat_q<- scalar_t,- QK8_0,- QR8_0,- QI8_0,- false,- block_q8_0,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q8_0<mmq_y>,- load_tiles_q8_0<mmq_y, nwarps, need_check>,- VDR_Q8_0_Q8_1_MMQ,- vec_dot_q8_0_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q8_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- const int mmq_x = MMQ_X_Q8_0;- const int mmq_y = MMQ_Y_Q8_0;- const int nwarps = NWARPS_Q8_0;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q8_0<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q8_0<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q2_K 64-#define MMQ_Y_Q2_K 128-#define NWARPS_Q2_K 8-#else-#define MMQ_X_Q2_K 4-#define MMQ_Y_Q2_K 32-#define NWARPS_Q2_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q2_K, 2)-#endif- mul_mat_q2_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q2_K;- const int mmq_y = MMQ_Y_Q2_K;- const int nwarps = NWARPS_Q2_K;-- mul_mat_q<- scalar_t,- QK_K,- QR2_K,- QI2_K,- false,- block_q2_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q2_K<mmq_y>,- load_tiles_q2_K<mmq_y, nwarps, need_check>,- VDR_Q2_K_Q8_1_MMQ,- vec_dot_q2_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q2_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- const int mmq_x = MMQ_X_Q2_K;- const int mmq_y = MMQ_Y_Q2_K;- const int nwarps = NWARPS_Q2_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q2_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q2_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q3_K 64-#define MMQ_Y_Q3_K 128-#define NWARPS_Q3_K 8-#else-#define MMQ_X_Q3_K 4-#define MMQ_Y_Q3_K 32-#define NWARPS_Q3_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q3_K, 2)-#endif- mul_mat_q3_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {-- const int mmq_x = MMQ_X_Q3_K;- const int mmq_y = MMQ_Y_Q3_K;- const int nwarps = NWARPS_Q3_K;-- mul_mat_q<- scalar_t,- QK_K,- QR3_K,- QI3_K,- false,- block_q3_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q3_K<mmq_y>,- load_tiles_q3_K<mmq_y, nwarps, need_check>,- VDR_Q3_K_Q8_1_MMQ,- vec_dot_q3_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q3_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- const int mmq_x = MMQ_X_Q3_K;- const int mmq_y = MMQ_Y_Q3_K;- const int nwarps = NWARPS_Q3_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q3_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q3_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q4_K 64-#define MMQ_Y_Q4_K 128-#define NWARPS_Q4_K 8-#else-#define MMQ_X_Q4_K 4-#define MMQ_Y_Q4_K 32-#define NWARPS_Q4_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_K, 2)-#endif- mul_mat_q4_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q4_K;- const int mmq_y = MMQ_Y_Q4_K;- const int nwarps = NWARPS_Q4_K;-- mul_mat_q<- scalar_t,- QK_K,- QR4_K,- QI4_K,- true,- block_q4_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q4_K<mmq_y>,- load_tiles_q4_K<mmq_y, nwarps, need_check>,- VDR_Q4_K_Q8_1_MMQ,- vec_dot_q4_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q4_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- const int mmq_x = MMQ_X_Q4_K;- const int mmq_y = MMQ_Y_Q4_K;- const int nwarps = NWARPS_Q4_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q4_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q4_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q5_K 64-#define MMQ_Y_Q5_K 128-#define NWARPS_Q5_K 8-#else-#define MMQ_X_Q5_K 4-#define MMQ_Y_Q5_K 32-#define NWARPS_Q5_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_K, 2)-#endif- mul_mat_q5_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q5_K;- const int mmq_y = MMQ_Y_Q5_K;- const int nwarps = NWARPS_Q5_K;-- mul_mat_q<- scalar_t,- QK_K,- QR5_K,- QI5_K,- true,- block_q5_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q5_K<mmq_y>,- load_tiles_q5_K<mmq_y, nwarps, need_check>,- VDR_Q5_K_Q8_1_MMQ,- vec_dot_q5_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q5_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- const int mmq_x = MMQ_X_Q5_K;- const int mmq_y = MMQ_Y_Q5_K;- const int nwarps = NWARPS_Q5_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q5_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q5_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}--#if defined(USE_ROCM)-#define MMQ_X_Q6_K 64-#define MMQ_Y_Q6_K 128-#define NWARPS_Q6_K 8-#else-#define MMQ_X_Q6_K 4-#define MMQ_Y_Q6_K 32-#define NWARPS_Q6_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q6_K, 2)-#endif- mul_mat_q6_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst) {- const int mmq_x = MMQ_X_Q6_K;- const int mmq_y = MMQ_Y_Q6_K;- const int nwarps = NWARPS_Q6_K;-- mul_mat_q<- scalar_t,- QK_K,- QR6_K,- QI6_K,- false,- block_q6_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q6_K<mmq_y>,- load_tiles_q6_K<mmq_y, nwarps, need_check>,- VDR_Q6_K_Q8_1_MMQ,- vec_dot_q6_K_q8_1_mul_mat>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);-}--template <typename scalar_t>-static void ggml_mul_mat_q6_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- cudaStream_t stream) {- const int mmq_x = MMQ_X_Q6_K;- const int mmq_y = MMQ_Y_Q6_K;- const int nwarps = NWARPS_Q6_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- const bool need_check = false;- mul_mat_q6_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- } else {- const bool need_check = true;- mul_mat_q6_K<scalar_t, need_check>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);- }-}diff --git a/python/freetoken/kernel/csrc/gguf/mmvq.cuh b/python/freetoken/kernel/csrc/gguf/mmvq.cuhdeleted file mode 100644index 7331731a..00000000--- a/python/freetoken/kernel/csrc/gguf/mmvq.cuh+++ /dev/null@@ -1,352 +0,0 @@-// copied from-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/mmvq.cuh-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu-template <typename scalar_t, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>-static __global__ void mul_mat_vec_q(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int ncols,- const int nrows,- const int nvecs) {- const auto row = blockIdx.x * blockDim.y + threadIdx.y;- const auto vec = blockIdx.y;-- if (row >= nrows || vec >= nvecs) {- return;- }-- const int blocks_per_row = ncols / qk;- const int blocks_per_warp = vdr * WARP_SIZE / qi;- const int nrows_y = (ncols + 512 - 1) / 512 * 512;-- // partial sum for each thread- float tmp = 0.0f;-- const block_q_t* x = (const block_q_t*)vx;- const block_q8_1* y = (const block_q8_1*)vy;-- for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) {- const int ibx = row * blocks_per_row + i; // x block index-- const int iby = vec * (nrows_y / QK8_1) + i * (qk / QK8_1); // y block index that aligns with ibx-- const int iqs = vdr * (threadIdx.x % (qi / vdr)); // x block quant index when casting the quants to int-- tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);- }-- // sum up partial sums and write back result-#pragma unroll- for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {- tmp += SGLANG_SHFL_XOR_SYNC(uint32_t(-1), tmp, mask);- }-- if (threadIdx.x == 0) {- dst[vec * nrows + row] = tmp;- }-}--template <typename scalar_t>-static void mul_mat_vec_q4_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q4_1_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q5_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q5_1_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q8_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q2_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q3_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q4_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q5_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_q6_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq2_xxs_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq2_xs_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq2_s_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq3_xxs_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq1_s_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq1_m_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI1_M, block_iq1_m, 1, vec_dot_iq1_m_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq4_nl_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq4_xs_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}--template <typename scalar_t>-static void mul_mat_vec_iq3_s_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int ncols,- const int nrows,- const int nvecs,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, nvecs, 1);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- mul_mat_vec_q<scalar_t, QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, ncols, nrows, nvecs);-}diff --git a/python/freetoken/kernel/csrc/gguf/moe.cuh b/python/freetoken/kernel/csrc/gguf/moe.cuhdeleted file mode 100644index 91e434fa..00000000--- a/python/freetoken/kernel/csrc/gguf/moe.cuh+++ /dev/null@@ -1,1379 +0,0 @@-// copied from-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/moe.cuh-#include <cstdint>--/* Adapted from ./csrc/quantization/gguf/mmq.cuh- */-template <- typename scalar_t,- int qk,- int qr,- int qi,- bool need_sum,- typename block_q_t,- int mmq_x,- int mmq_y,- int nwarps,- allocate_tiles_cuda_t allocate_tiles,- load_tiles_cuda_t load_tiles,- int vdr,- vec_dot_q_mul_mat_cuda_t vec_dot>-static __device__ __forceinline__ void moe_q(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* __restrict__ sorted_token_ids,- const int* __restrict__ expert_ids,- const int* __restrict__ num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int blocks_per_row_x = ncols_x / qk;- const int blocks_per_col_y = nrows_y / QK8_1;- const int blocks_per_warp = WARP_SIZE_GGUF / qi;-- const int ncols_dst = ncols_y * top_k;-- const auto row_dst_0 = blockIdx.x * mmq_y;- const int& row_x_0 = row_dst_0;-- const auto col_dst_0 = blockIdx.y * mmq_x;-- int token_offs[mmq_x / nwarps];- for (int i = 0; i < mmq_x; i += nwarps) {- token_offs[i / nwarps] = sorted_token_ids[col_dst_0 + threadIdx.y + i];- }-- const int exp_idx = expert_ids[blockIdx.y];- if (exp_idx > 255 || exp_idx < 0) return;- if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;-- const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);- const block_q8_1* y = (const block_q8_1*)(vy);-- int* tile_x_ql = nullptr;- half2* tile_x_dm = nullptr;- int* tile_x_qh = nullptr;- int* tile_x_sc = nullptr;-- allocate_tiles(&tile_x_ql, &tile_x_dm, &tile_x_qh, &tile_x_sc);-- __shared__ int tile_y_qs[mmq_x * WARP_SIZE_GGUF];- __shared__ half2 tile_y_ds[mmq_x * WARP_SIZE_GGUF / QI8_1];-- float sum[mmq_y / WARP_SIZE_GGUF][mmq_x / nwarps] = {{0.0f}};-- for (int ib0 = 0; ib0 < blocks_per_row_x; ib0 += blocks_per_warp) {- load_tiles(- x + row_x_0 * blocks_per_row_x + ib0,- tile_x_ql,- tile_x_dm,- tile_x_qh,- tile_x_sc,- threadIdx.y,- nrows_x - row_x_0 - 1,- threadIdx.x,- blocks_per_row_x);-- const int n_per_r = ((qk * blocks_per_warp) / qr);-#pragma unroll- for (int ir = 0; ir < qr && ib0 * qk + ir * n_per_r < ncols_x; ++ir) {- const auto kqs = ir * WARP_SIZE_GGUF + threadIdx.x;- const int kbxd = kqs / QI8_1;--#pragma unroll- for (int i = 0; i < mmq_x; i += nwarps) {- const int col_y_eff = token_offs[i / nwarps] / top_k;- const int block_x = ib0 * (qk / QK8_1) + kbxd;- if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {- const block_q8_1* by0 = &y[col_y_eff * blocks_per_col_y + block_x];- const int index_y = (threadIdx.y + i) * WARP_SIZE_GGUF + kqs % WARP_SIZE_GGUF;- tile_y_qs[index_y] = get_int_from_int8_aligned(by0->qs, threadIdx.x % QI8_1);- }- }-- if (threadIdx.x < n_per_r / QK8_1) {- const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1);- const int col_y_eff = token_offs[threadIdx.y] / top_k;- const int block_x = ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby;-- if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {- const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds;- half2* dsi_dst = &tile_y_ds[threadIdx.y * (WARP_SIZE_GGUF / QI8_1) + kby];-- if (need_sum) {- *dsi_dst = *dsi_src;- } else {- float* dfi_dst = (float*)dsi_dst;- *dfi_dst = __low2float(*dsi_src);- }- }- }- __syncthreads();-- // #pragma unroll // unrolling this loop causes too much register pressure- for (int k = ir * WARP_SIZE_GGUF / qr; k < (ir + 1) * WARP_SIZE_GGUF / qr; k += vdr) {-#pragma unroll- for (int j = 0; j < mmq_x; j += nwarps) {-#pragma unroll- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {- sum[i / WARP_SIZE_GGUF][j / nwarps] += vec_dot(- tile_x_ql, tile_x_dm, tile_x_qh, tile_x_sc, tile_y_qs, tile_y_ds, threadIdx.x + i, threadIdx.y + j, k);- }- }- }- __syncthreads();- }- }--#pragma unroll- for (int j = 0; j < mmq_x; j += nwarps) {- const int col_dst = token_offs[j / nwarps];- if (col_dst >= ncols_dst) {- return;- }--#pragma unroll- for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {- const auto row_dst = row_dst_0 + threadIdx.x + i;- if (row_dst >= nrows_dst) {- continue;- }- dst[col_dst * nrows_dst + row_dst] = sum[i / WARP_SIZE_GGUF][j / nwarps];- }- }-}--#if defined(USE_ROCM)-#define MOE_X_Q4_0 8-#define MOE_Y_Q4_0 128-#define NWARPS_Q4_0 8-#else-#define MOE_X_Q4_0 4-#define MOE_Y_Q4_0 32-#define NWARPS_Q4_0 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_0, 2)-#endif- moe_q4_0(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q4_0;- const int mmq_y = MOE_Y_Q4_0;- const int nwarps = NWARPS_Q4_0;-- moe_q<- scalar_t,- QK4_0,- QR4_0,- QI4_0,- true,- block_q4_0,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q4_0<mmq_y>,- load_tiles_q4_0<mmq_y, nwarps, need_check>,- VDR_Q4_0_Q8_1_MMQ,- vec_dot_q4_0_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q4_0_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- int mmq_x = MOE_X_Q4_0;- int mmq_y = MOE_Y_Q4_0;- int nwarps = NWARPS_Q4_0;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q4_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q4_1 8-#define MOE_Y_Q4_1 128-#define NWARPS_Q4_1 8-#else-#define MOE_X_Q4_1 4-#define MOE_Y_Q4_1 32-#define NWARPS_Q4_1 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_1, 2)-#endif- moe_q4_1(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q4_1;- const int mmq_y = MOE_Y_Q4_1;- const int nwarps = NWARPS_Q4_1;-- moe_q<- scalar_t,- QK4_1,- QR4_1,- QI4_1,- true,- block_q4_1,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q4_1<mmq_y>,- load_tiles_q4_1<mmq_y, nwarps, need_check>,- VDR_Q4_1_Q8_1_MMQ,- vec_dot_q4_1_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q4_1_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- int mmq_x = MOE_X_Q4_1;- int mmq_y = MOE_Y_Q4_1;- int nwarps = NWARPS_Q4_1;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q4_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q5_0 8-#define MOE_Y_Q5_0 128-#define NWARPS_Q5_0 8-#else-#define MOE_X_Q5_0 4-#define MOE_Y_Q5_0 32-#define NWARPS_Q5_0 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_0, 2)-#endif- moe_q5_0(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q5_0;- const int mmq_y = MOE_Y_Q5_0;- const int nwarps = NWARPS_Q5_0;-- moe_q<- scalar_t,- QK5_0,- QR5_0,- QI5_0,- false,- block_q5_0,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q5_0<mmq_y>,- load_tiles_q5_0<mmq_y, nwarps, need_check>,- VDR_Q5_0_Q8_1_MMQ,- vec_dot_q5_0_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q5_0_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- const int mmq_x = MOE_X_Q5_0;- const int mmq_y = MOE_Y_Q5_0;- const int nwarps = NWARPS_Q5_0;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q5_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q5_1 8-#define MOE_Y_Q5_1 128-#define NWARPS_Q5_1 8-#else-#define MOE_X_Q5_1 4-#define MOE_Y_Q5_1 32-#define NWARPS_Q5_1 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_1, 2)-#endif- moe_q5_1(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q5_1;- const int mmq_y = MOE_Y_Q5_1;- const int nwarps = NWARPS_Q5_1;-- moe_q<- scalar_t,- QK5_1,- QR5_1,- QI5_1,- true,- block_q5_1,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q5_1<mmq_y>,- load_tiles_q5_1<mmq_y, nwarps, need_check>,- VDR_Q5_1_Q8_1_MMQ,- vec_dot_q5_1_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q5_1_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- const int mmq_x = MOE_X_Q5_1;- const int mmq_y = MOE_Y_Q5_1;- const int nwarps = NWARPS_Q5_1;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q5_1<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q8_0 8-#define MOE_Y_Q8_0 128-#define NWARPS_Q8_0 8-#else-#define MOE_X_Q8_0 4-#define MOE_Y_Q8_0 32-#define NWARPS_Q8_0 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q8_0, 2)-#endif- moe_q8_0(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q8_0;- const int mmq_y = MOE_Y_Q8_0;- const int nwarps = NWARPS_Q8_0;-- moe_q<- scalar_t,- QK8_0,- QR8_0,- QI8_0,- false,- block_q8_0,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q8_0<mmq_y>,- load_tiles_q8_0<mmq_y, nwarps, need_check>,- VDR_Q8_0_Q8_1_MMQ,- vec_dot_q8_0_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q8_0_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- const int mmq_x = MOE_X_Q8_0;- const int mmq_y = MOE_Y_Q8_0;- const int nwarps = NWARPS_Q8_0;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q8_0<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q2_K 8-#define MOE_Y_Q2_K 128-#define NWARPS_Q2_K 8-#else-#define MOE_X_Q2_K 4-#define MOE_Y_Q2_K 32-#define NWARPS_Q2_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q2_K, 2)-#endif- moe_q2_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q2_K;- const int mmq_y = MOE_Y_Q2_K;- const int nwarps = NWARPS_Q2_K;-- moe_q<- scalar_t,- QK_K,- QR2_K,- QI2_K,- false,- block_q2_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q2_K<mmq_y>,- load_tiles_q2_K<mmq_y, nwarps, need_check>,- VDR_Q2_K_Q8_1_MMQ,- vec_dot_q2_K_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q2_K_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- const int mmq_x = MOE_X_Q2_K;- const int mmq_y = MOE_Y_Q2_K;- const int nwarps = NWARPS_Q2_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q2_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q3_K 8-#define MOE_Y_Q3_K 128-#define NWARPS_Q3_K 8-#else-#define MOE_X_Q3_K 4-#define MOE_Y_Q3_K 32-#define NWARPS_Q3_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q3_K, 2)-#endif- moe_q3_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {-- const int mmq_x = MOE_X_Q3_K;- const int mmq_y = MOE_Y_Q3_K;- const int nwarps = NWARPS_Q3_K;-- moe_q<- scalar_t,- QK_K,- QR3_K,- QI3_K,- false,- block_q3_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q3_K<mmq_y>,- load_tiles_q3_K<mmq_y, nwarps, need_check>,- VDR_Q3_K_Q8_1_MMQ,- vec_dot_q3_K_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}-template <typename scalar_t>-static void ggml_moe_q3_K_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- const int mmq_x = MOE_X_Q3_K;- const int mmq_y = MOE_Y_Q3_K;- const int nwarps = NWARPS_Q3_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q3_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q4_K 8-#define MOE_Y_Q4_K 128-#define NWARPS_Q4_K 8-#else-#define MOE_X_Q4_K 4-#define MOE_Y_Q4_K 32-#define NWARPS_Q4_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q4_K, 2)-#endif- moe_q4_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q4_K;- const int mmq_y = MOE_Y_Q4_K;- const int nwarps = NWARPS_Q4_K;-- moe_q<- scalar_t,- QK_K,- QR4_K,- QI4_K,- true,- block_q4_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q4_K<mmq_y>,- load_tiles_q4_K<mmq_y, nwarps, need_check>,- VDR_Q4_K_Q8_1_MMQ,- vec_dot_q4_K_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q4_K_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- const int mmq_x = MOE_X_Q4_K;- const int mmq_y = MOE_Y_Q4_K;- const int nwarps = NWARPS_Q4_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q4_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q5_K 8-#define MOE_Y_Q5_K 128-#define NWARPS_Q5_K 8-#else-#define MOE_X_Q5_K 4-#define MOE_Y_Q5_K 32-#define NWARPS_Q5_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q5_K, 2)-#endif- moe_q5_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q5_K;- const int mmq_y = MOE_Y_Q5_K;- const int nwarps = NWARPS_Q5_K;-- moe_q<- scalar_t,- QK_K,- QR5_K,- QI5_K,- true,- block_q5_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q5_K<mmq_y>,- load_tiles_q5_K<mmq_y, nwarps, need_check>,- VDR_Q5_K_Q8_1_MMQ,- vec_dot_q5_K_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q5_K_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- const int mmq_x = MOE_X_Q5_K;- const int mmq_y = MOE_Y_Q5_K;- const int nwarps = NWARPS_Q5_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q5_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}--#if defined(USE_ROCM)-#define MOE_X_Q6_K 8-#define MOE_Y_Q6_K 128-#define NWARPS_Q6_K 8-#else-#define MOE_X_Q6_K 4-#define MOE_Y_Q6_K 32-#define NWARPS_Q6_K 4-#endif--template <typename scalar_t, bool need_check>-static __global__ void-#if defined(USE_ROCM)-__launch_bounds__(WARP_SIZE_GGUF* NWARPS_Q6_K, 2)-#endif- moe_q6_K(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k) {- const int mmq_x = MOE_X_Q6_K;- const int mmq_y = MOE_Y_Q6_K;- const int nwarps = NWARPS_Q6_K;-- moe_q<- scalar_t,- QK_K,- QR6_K,- QI6_K,- false,- block_q6_K,- mmq_x,- mmq_y,- nwarps,- allocate_tiles_q6_K<mmq_y>,- load_tiles_q6_K<mmq_y, nwarps, need_check>,- VDR_Q6_K_Q8_1_MMQ,- vec_dot_q6_K_q8_1_mul_mat>(- vx,- vy,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);-}--template <typename scalar_t>-static void ggml_moe_q6_K_q8_1_cuda(- const void* inp,- const void* w,- scalar_t* dst,- const int* sorted_token_ids,- const int* expert_ids,- const int* num_tokens_post_padded,- const int exp_stride,- const int ncols_x,- const int nrows_x,- const int ncols_y,- const int nrows_y,- const int nrows_dst,- const int top_k,- const int tokens_post_padded,- cudaStream_t stream) {- const int mmq_x = MOE_X_Q6_K;- const int mmq_y = MOE_Y_Q6_K;- const int nwarps = NWARPS_Q6_K;-- const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y;- const int block_num_y = (tokens_post_padded) / mmq_x;- const dim3 block_nums(block_num_x, block_num_y, 1);- const dim3 block_dims(WARP_SIZE_GGUF, nwarps, 1);-- if (nrows_x % mmq_y == 0) {- constexpr bool need_check = false;- moe_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- } else {- constexpr bool need_check = true;- moe_q6_K<scalar_t, need_check><<<block_nums, block_dims, 0, stream>>>(- w,- inp,- dst,- sorted_token_ids,- expert_ids,- num_tokens_post_padded,- exp_stride,- ncols_x,- nrows_x,- ncols_y,- nrows_y,- nrows_dst,- top_k);- }-}diff --git a/python/freetoken/kernel/csrc/gguf/moe_vec.cuh b/python/freetoken/kernel/csrc/gguf/moe_vec.cuhdeleted file mode 100644index 8cef9e08..00000000--- a/python/freetoken/kernel/csrc/gguf/moe_vec.cuh+++ /dev/null@@ -1,413 +0,0 @@-// copied from-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/moe_vec.cuh-// copied and adapted from-// https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmvq.cu-template <typename scalar_t, int qk, int qi, typename block_q_t, int vdr, vec_dot_q_cuda_t vec_dot_q_cuda>-static __global__ void moe_vec_q(- const void* __restrict__ vx,- const void* __restrict__ vy,- scalar_t* __restrict__ dst,- const int* topk_ids,- const int topk,- const int ncols,- const int nrows,- const int token_stride) {- const auto row = blockIdx.x * blockDim.y + threadIdx.y;-- const auto token = blockIdx.z / topk;- const auto expert = (topk_ids)[blockIdx.z];-- if (row >= nrows) {- return;- }-- const int blocks_per_row = ncols / qk;- const int blocks_per_warp = vdr * WARP_SIZE / qi;-- // partial sum for each thread- float tmp = 0.0f;-- const block_q_t* x = ((const block_q_t*)vx) + expert * nrows * blocks_per_row;- const block_q8_1* y = (const block_q8_1*)(((const int*)vy) + token * token_stride);-- for (auto i = threadIdx.x / (qi / vdr); i < blocks_per_row; i += blocks_per_warp) {- const int ibx = row * blocks_per_row + i; // x block index-- const int iby = i * (qk / QK8_1); // y block index that aligns with ibx-- const int iqs = vdr * (threadIdx.x % (qi / vdr)); // x block quant index when casting the quants to int-- tmp += vec_dot_q_cuda(&x[ibx], &y[iby], iqs);- }-- // sum up partial sums and write back result-#pragma unroll- for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {- tmp += SGLANG_SHFL_XOR_SYNC(uint32_t(-1), tmp, mask);- }-- if (threadIdx.x == 0) {- dst[blockIdx.z * nrows + row] = tmp;- }-}--template <typename scalar_t>-static void moe_vec_q4_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK4_0, QI4_0, block_q4_0, VDR_Q4_0_Q8_1_MMVQ, vec_dot_q4_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q4_1_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK4_0, QI4_1, block_q4_1, VDR_Q4_1_Q8_1_MMVQ, vec_dot_q4_1_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q5_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK5_0, QI5_0, block_q5_0, VDR_Q5_0_Q8_1_MMVQ, vec_dot_q5_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q5_1_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK5_1, QI5_1, block_q5_1, VDR_Q5_1_Q8_1_MMVQ, vec_dot_q5_1_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q8_0_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK8_0, QI8_0, block_q8_0, VDR_Q8_0_Q8_1_MMVQ, vec_dot_q8_0_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q2_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI2_K, block_q2_K, VDR_Q2_K_Q8_1_MMVQ, vec_dot_q2_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q3_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI3_K, block_q3_K, VDR_Q3_K_Q8_1_MMVQ, vec_dot_q3_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q4_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI4_K, block_q4_K, VDR_Q4_K_Q8_1_MMVQ, vec_dot_q4_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q5_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI5_K, block_q5_K, VDR_Q5_K_Q8_1_MMVQ, vec_dot_q5_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_q6_K_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI6_K, block_q6_K, VDR_Q6_K_Q8_1_MMVQ, vec_dot_q6_K_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq2_xxs_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI2_XXS, block_iq2_xxs, 1, vec_dot_iq2_xxs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq2_xs_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI2_XS, block_iq2_xs, 1, vec_dot_iq2_xs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq2_s_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI2_S, block_iq2_s, 1, vec_dot_iq2_s_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq3_xxs_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI3_XXS, block_iq3_xxs, 1, vec_dot_iq3_xxs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq1_s_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI1_S, block_iq1_s, 1, vec_dot_iq1_s_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq1_m_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI1_M, block_iq1_m, 1, vec_dot_iq1_m_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq4_nl_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK4_NL, QI4_NL, block_iq4_nl, VDR_Q4_0_Q8_1_MMVQ, vec_dot_iq4_nl_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq4_xs_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI4_XS, block_iq4_xs, 1, vec_dot_iq4_xs_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}--template <typename scalar_t>-static void moe_vec_iq3_s_q8_1_cuda(- const void* vx,- const void* vy,- scalar_t* dst,- const int* topk_ids,- const int top_k,- const int tokens,- const int ncols,- const int nrows,- const int token_stride,- cudaStream_t stream) {- const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y;- const dim3 block_nums(block_num_y, 1, tokens * top_k);- const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1);- moe_vec_q<scalar_t, QK_K, QI3_XS, block_iq3_s, 1, vec_dot_iq3_s_q8_1>- <<<block_nums, block_dims, 0, stream>>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride);-}diff --git a/python/freetoken/kernel/csrc/gguf/vecdotq.cuh b/python/freetoken/kernel/csrc/gguf/vecdotq.cuhdeleted file mode 100644index 08b4cd26..00000000--- a/python/freetoken/kernel/csrc/gguf/vecdotq.cuh+++ /dev/null@@ -1,2037 +0,0 @@-// copied from-// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/vecdotq.cuh-// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/vecdotq.cuh-// and https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/mmq.cu-static __device__ __forceinline__ int get_int_b2(const void* x, const int& i32) {- const uint16_t* x16 = (const uint16_t*)x; // assume at least 2 byte alignment-- int x32 = x16[2 * i32 + 0] << 0;- x32 |= x16[2 * i32 + 1] << 16;-- return x32;-}--static __device__ __forceinline__ int get_int_b4(const void* x, const int& i32) {- return ((const int*)x)[i32]; // assume at least 4 byte alignment-}--static __device__ __forceinline__ int get_int_from_int8(const int8_t* x8, const int& i32) {- const uint16_t* x16 = (const uint16_t*)(x8 + sizeof(int) * i32); // assume at least 2 byte alignment- int x32 = 0;- x32 |= x16[0] << 0;- x32 |= x16[1] << 16;- return x32;-}--static __device__ __forceinline__ int get_int_from_uint8(const uint8_t* x8, const int& i32) {- const uint16_t* x16 = (const uint16_t*)(x8 + sizeof(int) * i32); // assume at least 2 byte alignment- int x32 = 0;- x32 |= x16[0] << 0;- x32 |= x16[1] << 16;- return x32;-}--static __device__ __forceinline__ int get_int_from_int8_aligned(const int8_t* x8, const int& i32) {- return *((const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment-}--static __device__ __forceinline__ int get_int_from_uint8_aligned(const uint8_t* x8, const int& i32) {- return *((const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment-}--// VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called-// MMVQ = mul_mat_vec_q, MMQ = mul_mat_q--#define VDR_Q4_0_Q8_1_MMVQ 2-#define VDR_Q4_0_Q8_1_MMQ 4--template <int vdr>-static __device__ __forceinline__ float-vec_dot_q4_0_q8_1_impl(const int* v, const int* u, const float& d4, const half2& ds8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- int sumi = 0;--#pragma unroll- for (int i = 0; i < vdr; ++i) {- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;-- // SIMD dot product of quantized values- sumi = __dp4a(vi0, u[2 * i + 0], sumi);- sumi = __dp4a(vi1, u[2 * i + 1], sumi);- }-- const float2 ds8f = __half22float2(ds8);-- // second part effectively subtracts 8 from each quant value- return d4 * (sumi * ds8f.x - (8 * vdr / QI4_0) * ds8f.y);-#endif-}--#define VDR_Q4_1_Q8_1_MMVQ 2-#define VDR_Q4_1_Q8_1_MMQ 4--template <int vdr>-static __device__ __forceinline__ float-vec_dot_q4_1_q8_1_impl(const int* v, const int* u, const half2& dm4, const half2& ds8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- int sumi = 0;--#pragma unroll- for (int i = 0; i < vdr; ++i) {- const int vi0 = (v[i] >> 0) & 0x0F0F0F0F;- const int vi1 = (v[i] >> 4) & 0x0F0F0F0F;-- // SIMD dot product of quantized values- sumi = __dp4a(vi0, u[2 * i + 0], sumi);- sumi = __dp4a(vi1, u[2 * i + 1], sumi);- }-- const float2 tmp = __half22float2(__hmul2(dm4, ds8));- const float d4d8 = tmp.x;- const float m4s8 = tmp.y;-- // scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding it- return sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));-#endif-}--#define VDR_Q5_0_Q8_1_MMVQ 2-#define VDR_Q5_0_Q8_1_MMQ 4--template <int vdr>-static __device__ __forceinline__ float-vec_dot_q5_0_q8_1_impl(const int* vl, const int* vh, const int* u, const float& d5, const half2& ds8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- int sumi = 0;--#pragma unroll- for (int i = 0; i < vdr; ++i) {- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28- sumi = __dp4a(vi0, u[2 * i + 0], sumi); // SIMD dot product of quantized values-- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28- sumi = __dp4a(vi1, u[2 * i + 1], sumi); // SIMD dot product of quantized values- }-- const float2 ds8f = __half22float2(ds8);-- // second part effectively subtracts 16 from each quant value- return d5 * (sumi * ds8f.x - (16 * vdr / QI5_0) * ds8f.y);-#endif-}--#define VDR_Q5_1_Q8_1_MMVQ 2-#define VDR_Q5_1_Q8_1_MMQ 4--template <int vdr>-static __device__ __forceinline__ float-vec_dot_q5_1_q8_1_impl(const int* vl, const int* vh, const int* u, const half2& dm5, const half2& ds8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- int sumi = 0;--#pragma unroll- for (int i = 0; i < vdr; ++i) {- int vi0 = (vl[i] >> 0) & 0x0F0F0F0F; // lower 4 qs bits, still need qh as 5th bits- vi0 |= (vh[i] << 4) & 0x00000010; // 0 -> 4- vi0 |= (vh[i] << 11) & 0x00001000; // 1 -> 12- vi0 |= (vh[i] << 18) & 0x00100000; // 2 -> 20- vi0 |= (vh[i] << 25) & 0x10000000; // 3 -> 28- sumi = __dp4a(vi0, u[2 * i + 0], sumi); // SIMD dot product of quantized values-- int vi1 = (vl[i] >> 4) & 0x0F0F0F0F; // upper 4 qs bits, still need qh as 5th bits- vi1 |= (vh[i] >> 12) & 0x00000010; // 16 -> 4- vi1 |= (vh[i] >> 5) & 0x00001000; // 17 -> 12- vi1 |= (vh[i] << 2) & 0x00100000; // 18 -> 20- vi1 |= (vh[i] << 9) & 0x10000000; // 19 -> 28- sumi = __dp4a(vi1, u[2 * i + 1], sumi); // SIMD dot product of quantized values- }-- const float2 tmp = __half22float2(__hmul2(dm5, ds8));- const float d5d8 = tmp.x;- const float m5s8 = tmp.y;-- // scale second part of sum by QI5_1 / vdr to compensate for multiple threads adding it- return sumi * d5d8 + m5s8 / (QI5_1 / vdr);-#endif-}--#define VDR_Q8_0_Q8_1_MMVQ 2-#define VDR_Q8_0_Q8_1_MMQ 8--template <int vdr>-static __device__ __forceinline__ float-vec_dot_q8_0_q8_1_impl(const int* v, const int* u, const float& d8_0, const float& d8_1) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- int sumi = 0;--#pragma unroll- for (int i = 0; i < vdr; ++i) {- // SIMD dot product of quantized values- sumi = __dp4a(v[i], u[i], sumi);- }- return d8_0 * d8_1 * sumi;-#endif-}--template <int vdr>-static __device__ __forceinline__ float-vec_dot_q8_1_q8_1_impl(const int* v, const int* u, const half2& dm8, const half2& ds8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA-- int sumi = 0;--#pragma unroll- for (int i = 0; i < vdr; ++i) {- // SIMD dot product of quantized values- sumi = __dp4a(v[i], u[i], sumi);- }-- const float2 tmp = __half22float2(__hmul2(dm8, ds8));- const float d8d8 = tmp.x;- const float m8s8 = tmp.y;-- // scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding it- return sumi * d8d8 + m8s8 / (QI8_1 / vdr);-#endif-}--#define VDR_Q2_K_Q8_1_MMVQ 1-#define VDR_Q2_K_Q8_1_MMQ 2--// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(- const int& v,- const int* __restrict__ u,- const uint8_t* __restrict__ scales,- const half2& dm2,- const float* __restrict__ d8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- float sumf_d = 0.0f;- float sumf_m = 0.0f;--#pragma unroll- for (int i = 0; i < QR2_K; ++i) {- const int sc = scales[2 * i];-- const int vi = (v >> (2 * i)) & 0x03030303;-- sumf_d += d8[i] * (__dp4a(vi, u[i], 0) * (sc & 0xF)); // SIMD dot product-- // fill int with 4x m- int m = sc >> 4;- m |= m << 8;- m |= m << 16;- sumf_m += d8[i] * __dp4a(m, u[i], 0); // multiply constant q2_K part with sum of q8_1 values- }-- const float2 dm2f = __half22float2(dm2);-- return dm2f.x * sumf_d - dm2f.y * sumf_m;-#endif-}--static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(- const int* __restrict__ v,- const int* __restrict__ u,- const uint8_t* __restrict__ scales,- const half2& dm2,- const float& d8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- int sumi_d = 0;- int sumi_m = 0;--#pragma unroll- for (int i0 = 0; i0 < QI8_1; i0 += QI8_1 / 2) {- int sumi_d_sc = 0;-- const int sc = scales[i0 / (QI8_1 / 2)];-- // fill int with 4x m- int m = sc >> 4;- m |= m << 8;- m |= m << 16;--#pragma unroll- for (int i = i0; i < i0 + QI8_1 / 2; ++i) {- sumi_d_sc = __dp4a(v[i], u[i], sumi_d_sc); // SIMD dot product- sumi_m = __dp4a(m, u[i], sumi_m); // multiply sum of q8_1 values with m- }-- sumi_d += sumi_d_sc * (sc & 0xF);- }-- const float2 dm2f = __half22float2(dm2);-- return d8 * (dm2f.x * sumi_d - dm2f.y * sumi_m);-#endif-}--#define VDR_Q3_K_Q8_1_MMVQ 1-#define VDR_Q3_K_Q8_1_MMQ 2--// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(- const int& vl,- const int& vh,- const int* __restrict__ u,- const uint8_t* __restrict__ scales,- const int& scale_offset,- const float& d3,- const float* __restrict__ d8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA-- float sumf = 0.0f;--#pragma unroll- for (int i = 0; i < QR3_K; ++i) {- const int isc = scale_offset + 2 * i;-- const int isc_low = isc % (QK_K / 32);- const int sc_shift_low = 4 * (isc / (QK_K / 32));- const int sc_low = (scales[isc_low] >> sc_shift_low) & 0xF;-- const int isc_high = isc % (QK_K / 64);- const int sc_shift_high = 2 * (isc / (QK_K / 64));- const int sc_high = ((scales[(QK_K / 32) + isc_high] >> sc_shift_high) & 3) << 4;-- const int sc = (sc_low | sc_high) - 32;-- const int vil = (vl >> (2 * i)) & 0x03030303;-- const int vih = ((vh >> i) << 2) & 0x04040404;-- const int vi = __vsubss4(vil, vih);-- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product- }-- return d3 * sumf;-#endif-}--static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(- const int* __restrict__ v,- const int* __restrict__ u,- const int8_t* __restrict__ scales,- const float& d3,- const float& d8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- int sumi = 0;--#pragma unroll- for (int i0 = 0; i0 < QR3_K * VDR_Q3_K_Q8_1_MMQ; i0 += QI8_1 / 2) {- int sumi_sc = 0;-- for (int i = i0; i < i0 + QI8_1 / 2; ++i) {- sumi_sc = __dp4a(v[i], u[i], sumi_sc); // SIMD dot product- }-- sumi += sumi_sc * scales[i0 / (QI8_1 / 2)];- }-- return d3 * d8 * sumi;-#endif-}--#define VDR_Q4_K_Q8_1_MMVQ 2-#define VDR_Q4_K_Q8_1_MMQ 8--// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(- const int* __restrict__ v,- const int* __restrict__ u,- const uint8_t* __restrict__ sc,- const uint8_t* __restrict__ m,- const half2& dm4,- const float* __restrict__ d8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA-- float sumf_d = 0.0f;- float sumf_m = 0.0f;--#pragma unroll- for (int i = 0; i < QR4_K; ++i) {- const int v0i = (v[0] >> (4 * i)) & 0x0F0F0F0F;- const int v1i = (v[1] >> (4 * i)) & 0x0F0F0F0F;-- const int dot1 = __dp4a(v1i, u[2 * i + 1], __dp4a(v0i, u[2 * i + 0], 0)); // SIMD dot product- const int dot2 = __dp4a(0x01010101, u[2 * i + 1], __dp4a(0x01010101, u[2 * i + 0], 0)); // sum of u-- sumf_d += d8[i] * (dot1 * sc[i]);- sumf_m += d8[i] * (dot2 * m[i]); // multiply constant part of q4_K with sum of q8_1 values- }-- const float2 dm4f = __half22float2(dm4);- return dm4f.x * sumf_d - dm4f.y * sumf_m;-#endif-}--static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(- const int* __restrict__ v,- const int* __restrict__ u,- const uint8_t* __restrict__ sc,- const uint8_t* __restrict__ m,- const half2& dm4,- const half2* __restrict__ ds8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- float sumf_d = 0.0f;- float sumf_m = 0.0f;--#pragma unroll- for (int i = 0; i < QR4_K * VDR_Q4_K_Q8_1_MMQ / QI8_1; ++i) {- int sumi_d = 0;--#pragma unroll- for (int j = 0; j < QI8_1; ++j) {- sumi_d = __dp4a((v[j] >> (4 * i)) & 0x0F0F0F0F, u[i * QI8_1 + j], sumi_d); // SIMD dot product- }-- const float2 ds8f = __half22float2(ds8[i]);-- sumf_d += ds8f.x * (sc[i] * sumi_d);- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val- }-- const float2 dm4f = __half22float2(dm4);-- return dm4f.x * sumf_d - dm4f.y * sumf_m;-#endif-}--#define VDR_Q5_K_Q8_1_MMVQ 2-#define VDR_Q5_K_Q8_1_MMQ 8--static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(- const int* __restrict__ vl,- const int* __restrict__ vh,- const int* __restrict__ u,- const uint8_t* __restrict__ sc,- const uint8_t* __restrict__ m,- const half2& dm5,- const float* __restrict__ d8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA-- float sumf_d = 0.0f;- float sumf_m = 0.0f;--#pragma unroll- for (int i = 0; i < QR5_K; ++i) {- const int vl0i = (vl[0] >> (4 * i)) & 0x0F0F0F0F;- const int vl1i = (vl[1] >> (4 * i)) & 0x0F0F0F0F;-- const int vh0i = ((vh[0] >> i) << 4) & 0x10101010;- const int vh1i = ((vh[1] >> i) << 4) & 0x10101010;-- const int v0i = vl0i | vh0i;- const int v1i = vl1i | vh1i;-- const int dot1 = __dp4a(v0i, u[2 * i + 0], __dp4a(v1i, u[2 * i + 1], 0)); // SIMD dot product- const int dot2 = __dp4a(0x01010101, u[2 * i + 0], __dp4a(0x01010101, u[2 * i + 1], 0)); // sum of u-- sumf_d += d8[i] * (dot1 * sc[i]);- sumf_m += d8[i] * (dot2 * m[i]);- }-- const float2 dm5f = __half22float2(dm5);- return dm5f.x * sumf_d - dm5f.y * sumf_m;-#endif-}--static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(- const int* __restrict__ v,- const int* __restrict__ u,- const uint8_t* __restrict__ sc,- const uint8_t* __restrict__ m,- const half2& dm4,- const half2* __restrict__ ds8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- float sumf_d = 0.0f;- float sumf_m = 0.0f;--#pragma unroll- for (int i = 0; i < QR5_K * VDR_Q5_K_Q8_1_MMQ / QI8_1; ++i) {- int sumi_d = 0;--#pragma unroll- for (int j = 0; j < QI8_1; ++j) {- sumi_d = __dp4a(v[i * QI8_1 + j], u[i * QI8_1 + j], sumi_d); // SIMD dot product- }-- const float2 ds8f = __half22float2(ds8[i]);-- sumf_d += ds8f.x * (sc[i] * sumi_d);- sumf_m += ds8f.y * m[i]; // sum of q8_1 block * q4_K min val- }-- const float2 dm4f = __half22float2(dm4);-- return dm4f.x * sumf_d - dm4f.y * sumf_m;-#endif-}--#define VDR_Q6_K_Q8_1_MMVQ 1-#define VDR_Q6_K_Q8_1_MMQ 8--// contiguous v/x values-static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(- const int& vl,- const int& vh,- const int* __restrict__ u,- const int8_t* __restrict__ scales,- const float& d,- const float* __restrict__ d8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- float sumf = 0.0f;--#pragma unroll- for (int i = 0; i < QR6_K; ++i) {- const int sc = scales[4 * i];- const int vil = (vl >> (4 * i)) & 0x0F0F0F0F;- const int vih = ((vh >> (4 * i)) << 4) & 0x30303030;- const int vi = __vsubss4((vil | vih), 0x20202020); // vi = (vil | vih) - 32-- sumf += d8[i] * (__dp4a(vi, u[i], 0) * sc); // SIMD dot product- }-- return d * sumf;-#endif-}--static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(- const int* __restrict__ v,- const int* __restrict__ u,- const int8_t* __restrict__ sc,- const float& d6,- const float* __restrict__ d8) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- float sumf_d = 0.0f;--#pragma unroll- for (int i0 = 0; i0 < VDR_Q6_K_Q8_1_MMQ; i0 += 4) {- int2 sumi_d = {0, 0}; // 2 q6_K scales per q8_1 scale--#pragma unroll- for (int i = i0; i < i0 + 2; ++i) {- sumi_d.x = __dp4a(v[2 * i + 0], u[2 * i + 0], sumi_d.x); // SIMD dot product- sumi_d.x = __dp4a(v[2 * i + 1], u[2 * i + 1], sumi_d.x); // SIMD dot product-- sumi_d.y = __dp4a(v[2 * i + 4], u[2 * i + 4], sumi_d.y); // SIMD dot product- sumi_d.y = __dp4a(v[2 * i + 5], u[2 * i + 5], sumi_d.y); // SIMD dot product- }-- sumf_d += d8[i0 / 4] * (sc[i0 / 2 + 0] * sumi_d.x + sc[i0 / 2 + 1] * sumi_d.y);- }-- return d6 * sumf_d;-#endif-}--static __device__ __forceinline__ float-vec_dot_q4_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q4_0* bq4_0 = (const block_q4_0*)vbq;-- int v[VDR_Q4_0_Q8_1_MMVQ];- int u[2 * VDR_Q4_0_Q8_1_MMVQ];--#pragma unroll- for (int i = 0; i < VDR_Q4_0_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_uint8(bq4_0->qs, iqs + i);- u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_0);- }-- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMVQ>(v, u, __half2float(bq4_0->d), bq8_1->ds);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q4_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI4_0) + mmq_y / QI4_0];- *x_ql = tile_x_qs;- *x_dm = (half2*)tile_x_d;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q4_0(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI4_0;- const int kqsx = k % QI4_0;-- const block_q4_0* bx0 = (const block_q4_0*)vx;- float* x_dmf = (float*)x_dm;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;- if (need_check) {- i = min(i, i_max);- }- const block_q4_0* bxi = bx0 + i * blocks_per_row + kbx;- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);- // x_dmf[i * (WARP_SIZE_GGUF/QI4_0) + i / QI4_0 + kbx] = bxi->d;- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_0;- const int kbxd = k % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_0) {- int i = i0 + i_offset * QI4_0 + k / blocks_per_tile_x_row;- if (need_check) {- i = min(i, i_max);- }- const block_q4_0* bxi = bx0 + i * blocks_per_row + kbxd;- x_dmf[i * (WARP_SIZE_GGUF / QI4_0) + i / QI4_0 + kbxd] = __half2float(bxi->d);- }-}--static __device__ __forceinline__ float vec_dot_q4_0_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- (void)x_qh;- (void)x_sc;-- const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2));- const float* x_dmf = (const float*)x_dm;-- int u[2 * VDR_Q4_0_Q8_1_MMQ];--#pragma unroll- for (int l = 0; l < VDR_Q4_0_Q8_1_MMQ; ++l) {- u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];- u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_0) % WARP_SIZE_GGUF];- }-- return vec_dot_q4_0_q8_1_impl<VDR_Q4_0_Q8_1_MMQ>(- &x_ql[i * (WARP_SIZE_GGUF + 1) + k],- u,- x_dmf[i * (WARP_SIZE_GGUF / QI4_0) + i / QI4_0 + k / QI4_0],- y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]);-}--static __device__ __forceinline__ float-vec_dot_q4_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q4_1* bq4_1 = (const block_q4_1*)vbq;-- int v[VDR_Q4_1_Q8_1_MMVQ];- int u[2 * VDR_Q4_1_Q8_1_MMVQ];--#pragma unroll- for (int i = 0; i < VDR_Q4_1_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_uint8_aligned(bq4_1->qs, iqs + i);- u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI4_1);- }-- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMVQ>(v, u, bq4_1->dm, bq8_1->ds);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q4_1(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + +mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI4_1) + mmq_y / QI4_1];- *x_ql = tile_x_qs;- *x_dm = tile_x_dm;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q4_1(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI4_1;- const int kqsx = k % QI4_1;-- const block_q4_1* bx0 = (const block_q4_1*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;- if (need_check) {- i = min(i, i_max);- }- const block_q4_1* bxi = bx0 + i * blocks_per_row + kbx;- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_1;- const int kbxd = k % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_1) {- int i = i0 + i_offset * QI4_1 + k / blocks_per_tile_x_row;- if (need_check) {- i = min(i, i_max);- }- const block_q4_1* bxi = bx0 + i * blocks_per_row + kbxd;- x_dm[i * (WARP_SIZE_GGUF / QI4_1) + i / QI4_1 + kbxd] = bxi->dm;- }-}--static __device__ __forceinline__ float vec_dot_q4_1_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2));-- int u[2 * VDR_Q4_1_Q8_1_MMQ];--#pragma unroll- for (int l = 0; l < VDR_Q4_1_Q8_1_MMQ; ++l) {- u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];- u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI4_1) % WARP_SIZE_GGUF];- }-- return vec_dot_q4_1_q8_1_impl<VDR_Q4_1_Q8_1_MMQ>(- &x_ql[i * (WARP_SIZE_GGUF + 1) + k],- u,- x_dm[i * (WARP_SIZE_GGUF / QI4_1) + i / QI4_1 + k / QI4_1],- y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]);-}--static __device__ __forceinline__ float-vec_dot_q5_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q5_0* bq5_0 = (const block_q5_0*)vbq;-- int vl[VDR_Q5_0_Q8_1_MMVQ];- int vh[VDR_Q5_0_Q8_1_MMVQ];- int u[2 * VDR_Q5_0_Q8_1_MMVQ];--#pragma unroll- for (int i = 0; i < VDR_Q5_0_Q8_1_MMVQ; ++i) {- vl[i] = get_int_from_uint8(bq5_0->qs, iqs + i);- vh[i] = get_int_from_uint8(bq5_0->qh, 0) >> (4 * (iqs + i));- u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_0);- }-- return vec_dot_q5_0_q8_1_impl<VDR_Q5_0_Q8_1_MMVQ>(vl, vh, u, __half2float(bq5_0->d), bq8_1->ds);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q5_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y];- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI5_0) + mmq_y / QI5_0];-- *x_ql = tile_x_ql;- *x_dm = (half2*)tile_x_d;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q5_0(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI5_0;- const int kqsx = k % QI5_0;-- const block_q5_0* bx0 = (const block_q5_0*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;-- if (need_check) {- i = min(i, i_max);- }- const block_q5_0* bxi = bx0 + i * blocks_per_row + kbx;- const int ql = get_int_from_uint8(bxi->qs, kqsx);- const int qh = get_int_from_uint8(bxi->qh, 0) >> (4 * (k % QI5_0));-- int qs0 = (ql >> 0) & 0x0F0F0F0F;- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28- qs0 = __vsubss4(qs0, 0x10101010); // subtract 16-- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 0] = qs0;-- int qs1 = (ql >> 4) & 0x0F0F0F0F;- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28- qs1 = __vsubss4(qs1, 0x10101010); // subtract 16-- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 1] = qs1;- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_0;- const int kbxd = k % blocks_per_tile_x_row;- float* x_dmf = (float*)x_dm;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_0) {- int i = i0 + i_offset * QI5_0 + k / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_0* bxi = bx0 + i * blocks_per_row + kbxd;- x_dmf[i * (WARP_SIZE_GGUF / QI5_0) + i / QI5_0 + kbxd] = __half2float(bxi->d);- }-}--static __device__ __forceinline__ float vec_dot_q5_0_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2));- const int index_bx = i * (WARP_SIZE_GGUF / QI5_0) + i / QI5_0 + k / QI5_0;- const float* x_dmf = (const float*)x_dm;- const float* y_df = (const float*)y_ds;-- int u[2 * VDR_Q5_0_Q8_1_MMQ];--#pragma unroll- for (int l = 0; l < VDR_Q5_0_Q8_1_MMQ; ++l) {- u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];- u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_0) % WARP_SIZE_GGUF];- }-- return vec_dot_q8_0_q8_1_impl<QR5_0 * VDR_Q5_0_Q8_1_MMQ>(- &x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k],- u,- x_dmf[index_bx],- y_df[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]);-}--static __device__ __forceinline__ float-vec_dot_q5_1_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q5_1* bq5_1 = (const block_q5_1*)vbq;-- int vl[VDR_Q5_1_Q8_1_MMVQ];- int vh[VDR_Q5_1_Q8_1_MMVQ];- int u[2 * VDR_Q5_1_Q8_1_MMVQ];--#pragma unroll- for (int i = 0; i < VDR_Q5_1_Q8_1_MMVQ; ++i) {- vl[i] = get_int_from_uint8_aligned(bq5_1->qs, iqs + i);- vh[i] = get_int_from_uint8_aligned(bq5_1->qh, 0) >> (4 * (iqs + i));- u[2 * i + 0] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- u[2 * i + 1] = get_int_from_int8_aligned(bq8_1->qs, iqs + i + QI5_1);- }-- return vec_dot_q5_1_q8_1_impl<VDR_Q5_1_Q8_1_MMVQ>(vl, vh, u, bq5_1->dm, bq8_1->ds);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q5_1(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI5_1) + mmq_y / QI5_1];-- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q5_1(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI5_1;- const int kqsx = k % QI5_1;-- const block_q5_1* bx0 = (const block_q5_1*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_1* bxi = bx0 + i * blocks_per_row + kbx;-- const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);- const int qh = get_int_from_uint8_aligned(bxi->qh, 0) >> (4 * (k % QI5_1));-- int qs0 = (ql >> 0) & 0x0F0F0F0F;- qs0 |= (qh << 4) & 0x00000010; // 0 -> 4- qs0 |= (qh << 11) & 0x00001000; // 1 -> 12- qs0 |= (qh << 18) & 0x00100000; // 2 -> 20- qs0 |= (qh << 25) & 0x10000000; // 3 -> 28-- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 0] = qs0;-- int qs1 = (ql >> 4) & 0x0F0F0F0F;- qs1 |= (qh >> 12) & 0x00000010; // 16 -> 4- qs1 |= (qh >> 5) & 0x00001000; // 17 -> 12- qs1 |= (qh << 2) & 0x00100000; // 18 -> 20- qs1 |= (qh << 9) & 0x10000000; // 19 -> 28-- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k + 1] = qs1;- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_1;- const int kbxd = k % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_1) {- int i = i0 + i_offset * QI5_1 + k / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_1* bxi = bx0 + i * blocks_per_row + kbxd;-- x_dm[i * (WARP_SIZE_GGUF / QI5_1) + i / QI5_1 + kbxd] = bxi->dm;- }-}--static __device__ __forceinline__ float vec_dot_q5_1_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- const int kyqs = k % (QI8_1 / 2) + QI8_1 * (k / (QI8_1 / 2));- const int index_bx = i * (WARP_SIZE_GGUF / QI5_1) + +i / QI5_1 + k / QI5_1;-- int u[2 * VDR_Q5_1_Q8_1_MMQ];--#pragma unroll- for (int l = 0; l < VDR_Q5_1_Q8_1_MMQ; ++l) {- u[2 * l + 0] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l) % WARP_SIZE_GGUF];- u[2 * l + 1] = y_qs[j * WARP_SIZE_GGUF + (kyqs + l + QI5_1) % WARP_SIZE_GGUF];- }-- return vec_dot_q8_1_q8_1_impl<QR5_1 * VDR_Q5_1_Q8_1_MMQ>(- &x_ql[i * (2 * WARP_SIZE_GGUF + 1) + 2 * k],- u,- x_dm[index_bx],- y_ds[j * (WARP_SIZE_GGUF / QI8_1) + (2 * k / QI8_1) % (WARP_SIZE_GGUF / QI8_1)]);-}--static __device__ __forceinline__ float-vec_dot_q8_0_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q8_0* bq8_0 = (const block_q8_0*)vbq;-- int v[VDR_Q8_0_Q8_1_MMVQ];- int u[VDR_Q8_0_Q8_1_MMVQ];--#pragma unroll- for (int i = 0; i < VDR_Q8_0_Q8_1_MMVQ; ++i) {- v[i] = get_int_from_int8(bq8_0->qs, iqs + i);- u[i] = get_int_from_int8_aligned(bq8_1->qs, iqs + i);- }-- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMVQ>(v, u, __half2float(bq8_0->d), __low2float(bq8_1->ds));-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q8_0(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_qs[mmq_y * (WARP_SIZE_GGUF) + mmq_y];- __shared__ float tile_x_d[mmq_y * (WARP_SIZE_GGUF / QI8_0) + mmq_y / QI8_0];-- *x_ql = tile_x_qs;- *x_dm = (half2*)tile_x_d;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q8_0(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI8_0;- const int kqsx = k % QI8_0;- float* x_dmf = (float*)x_dm;-- const block_q8_0* bx0 = (const block_q8_0*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;-- if (need_check) {- i = min(i, i_max);- }- const block_q8_0* bxi = bx0 + i * blocks_per_row + kbx;- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_int8(bxi->qs, kqsx);- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI8_0;- const int kbxd = k % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI8_0) {- int i = i0 + i_offset * QI8_0 + k / blocks_per_tile_x_row;-- if (need_check) {- i = min(i, i_max);- }- const block_q8_0* bxi = bx0 + i * blocks_per_row + kbxd;- x_dmf[i * (WARP_SIZE_GGUF / QI8_0) + i / QI8_0 + kbxd] = __half2float(bxi->d);- }-}--static __device__ __forceinline__ float vec_dot_q8_0_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- const float* x_dmf = (const float*)x_dm;- const float* y_df = (const float*)y_ds;-- return vec_dot_q8_0_q8_1_impl<VDR_Q8_0_Q8_1_MMQ>(- &x_ql[i * (WARP_SIZE_GGUF + 1) + k],- &y_qs[j * WARP_SIZE_GGUF + k],- x_dmf[i * (WARP_SIZE_GGUF / QI8_0) + i / QI8_0 + k / QI8_0],- y_df[j * (WARP_SIZE_GGUF / QI8_1) + k / QI8_1]);-}--static __device__ __forceinline__ float-vec_dot_q2_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q2_K* bq2_K = (const block_q2_K*)vbq;-- const int bq8_offset = QR2_K * (iqs / QI8_1);- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2);-- const uint8_t* scales = bq2_K->scales + scale_offset;-- const int v = get_int_from_uint8_aligned(bq2_K->qs, iqs);- int u[QR2_K];- float d8[QR2_K];--#pragma unroll- for (int i = 0; i < QR2_K; ++i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);- d8[i] = __low2float(bq8_1[bq8_offset + i].ds);- }-- return vec_dot_q2_K_q8_1_impl_mmvq(v, u, scales, bq2_K->dm, d8);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q2_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI2_K) + mmq_y / QI2_K];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 4) + mmq_y / 4];-- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_sc = tile_x_sc;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q2_K(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI2_K;- const int kqsx = k % QI2_K;-- const block_q2_K* bx0 = (const block_q2_K*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;-- if (need_check) {- i = min(i, i_max);- }- const block_q2_K* bxi = bx0 + i * blocks_per_row + kbx;- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI2_K;- const int kbxd = k % blocks_per_tile_x_row;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI2_K) {- int i = (i0 + i_offset * QI2_K + k / blocks_per_tile_x_row) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }- const block_q2_K* bxi = bx0 + i * blocks_per_row + kbxd;- x_dm[i * (WARP_SIZE_GGUF / QI2_K) + i / QI2_K + kbxd] = bxi->dm;- }--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {- int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF / 4);-- if (need_check) {- i = min(i, i_max);- }- const block_q2_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 4)) / (QI2_K / 4);- x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + k % (WARP_SIZE_GGUF / 4)] =- get_int_from_uint8_aligned(bxi->scales, k % (QI2_K / 4));- }-}--static __device__ __forceinline__ float vec_dot_q2_K_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- const int kbx = k / QI2_K;- const int ky = (k % QI2_K) * QR2_K;- const float* y_df = (const float*)y_ds;-- int v[QR2_K * VDR_Q2_K_Q8_1_MMQ];-- const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx * QI2_K + (QI2_K / 2) * (ky / (2 * QI2_K)) + ky % (QI2_K / 2);- const int shift = 2 * ((ky % (2 * QI2_K)) / (QI2_K / 2));--#pragma unroll- for (int l = 0; l < QR2_K * VDR_Q2_K_Q8_1_MMQ; ++l) {- v[l] = (x_ql[kqsx + l] >> shift) & 0x03030303;- }-- const uint8_t* scales = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + kbx * 4]) + ky / 4;-- const int index_y = j * WARP_SIZE_GGUF + (QR2_K * k) % WARP_SIZE_GGUF;- return vec_dot_q2_K_q8_1_impl_mmq(- v, &y_qs[index_y], scales, x_dm[i * (WARP_SIZE_GGUF / QI2_K) + i / QI2_K + kbx], y_df[index_y / QI8_1]);-}--static __device__ __forceinline__ float-vec_dot_q3_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q3_K* bq3_K = (const block_q3_K*)vbq;-- const int bq8_offset = QR3_K * (iqs / (QI3_K / 2));- const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1 / 2);-- const float d = __half2float(bq3_K->d);-- const int vl = get_int_from_uint8(bq3_K->qs, iqs);-- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted- const int vh = ~get_int_from_uint8(bq3_K->hmask, iqs % (QI3_K / 2)) >> bq8_offset;-- int u[QR3_K];- float d8[QR3_K];--#pragma unroll- for (int i = 0; i < QR3_K; ++i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + i].qs, iqs % QI8_1);- d8[i] = __low2float(bq8_1[bq8_offset + i].ds);- }-- return vec_dot_q3_K_q8_1_impl_mmvq(vl, vh, u, bq3_K->scales, scale_offset, d, d8);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q3_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI3_K) + mmq_y / QI3_K];- __shared__ int tile_x_qh[mmq_y * (WARP_SIZE_GGUF / 2) + mmq_y / 2];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 4) + mmq_y / 4];-- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_qh = tile_x_qh;- *x_sc = tile_x_sc;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q3_K(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI3_K;- const int kqsx = k % QI3_K;-- const block_q3_K* bx0 = (const block_q3_K*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;- if (need_check) {- i = min(i, i_max);- }- const block_q3_K* bxi = bx0 + i * blocks_per_row + kbx;- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8(bxi->qs, kqsx);- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI3_K;- const int kbxd = k % blocks_per_tile_x_row;- float* x_dmf = (float*)x_dm;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI3_K) {- int i = (i0 + i_offset * QI3_K + k / blocks_per_tile_x_row) % mmq_y;- if (need_check) {- i = min(i, i_max);- }- const block_q3_K* bxi = bx0 + i * blocks_per_row + kbxd;- x_dmf[i * (WARP_SIZE_GGUF / QI3_K) + i / QI3_K + kbxd] = __half2float(bxi->d);- }--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 2) {- int i = i0 + i_offset * 2 + k / (WARP_SIZE_GGUF / 2);- if (need_check) {- i = min(i, i_max);- }- const block_q3_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 2)) / (QI3_K / 2);- // invert the mask with ~ so that a 0/1 results in 4/0 being subtracted- x_qh[i * (WARP_SIZE_GGUF / 2) + i / 2 + k % (WARP_SIZE_GGUF / 2)] =- ~get_int_from_uint8(bxi->hmask, k % (QI3_K / 2));- }--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 4) {- int i = i0 + i_offset * 4 + k / (WARP_SIZE_GGUF / 4);- if (need_check) {- i = min(i, i_max);- }- const block_q3_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 4)) / (QI3_K / 4);-- const int ksc = k % (QI3_K / 4);-- const int ksc_low = ksc % (QI3_K / 8);- const int shift_low = 4 * (ksc / (QI3_K / 8));- const int sc_low = (get_int_from_uint8(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F;-- const int ksc_high = QI3_K / 8;- const int shift_high = 2 * ksc;- const int sc_high = ((get_int_from_uint8(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030;-- const int sc = __vsubss4(sc_low | sc_high, 0x20202020);-- x_sc[i * (WARP_SIZE_GGUF / 4) + i / 4 + k % (WARP_SIZE_GGUF / 4)] = sc;- }-}--static __device__ __forceinline__ float vec_dot_q3_K_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- const int kbx = k / QI3_K;- const int ky = (k % QI3_K) * QR3_K;- const float* x_dmf = (const float*)x_dm;- const float* y_df = (const float*)y_ds;-- const int8_t* scales = ((const int8_t*)(x_sc + i * (WARP_SIZE_GGUF / 4) + i / 4 + kbx * 4)) + ky / 4;-- int v[QR3_K * VDR_Q3_K_Q8_1_MMQ];--#pragma unroll- for (int l = 0; l < QR3_K * VDR_Q3_K_Q8_1_MMQ; ++l) {- const int kqsx = i * (WARP_SIZE_GGUF + 1) + kbx * QI3_K + (QI3_K / 2) * (ky / (2 * QI3_K)) + ky % (QI3_K / 2);- const int shift = 2 * ((ky % 32) / 8);- const int vll = (x_ql[kqsx + l] >> shift) & 0x03030303;-- const int vh = x_qh[i * (WARP_SIZE_GGUF / 2) + i / 2 + kbx * (QI3_K / 2) + (ky + l) % 8] >> ((ky + l) / 8);- const int vlh = (vh << 2) & 0x04040404;-- v[l] = __vsubss4(vll, vlh);- }-- const int index_y = j * WARP_SIZE_GGUF + (k * QR3_K) % WARP_SIZE_GGUF;- return vec_dot_q3_K_q8_1_impl_mmq(- v, &y_qs[index_y], scales, x_dmf[i * (WARP_SIZE_GGUF / QI3_K) + i / QI3_K + kbx], y_df[index_y / QI8_1]);-}--static __device__ __forceinline__ float-vec_dot_q4_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q4_K* bq4_K = (const block_q4_K*)vbq;-- int v[2];- int u[2 * QR4_K];- float d8[QR4_K];-- // iqs is in 0,2..30. bq8_offset = iqs/4 -> bq8_offset = 0, 2, 4, 6- const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2));-- // iqs = 0....3 -> bq8_offset = 0, want q4_offset = 0, 4, 8, 12- // iqs = 4....7 -> bq8_offset = 2, want q4_offset = 32, 36, 40, 44- // iqs = 8...11 -> bq8_offset = 4, want q4_offset = 64, 68, 72, 76- // iqs = 12..15 -> bq8_offset = 6, want q4_offset = 96, 100, 104, 108-- const int* q4 = (const int*)(bq4_K->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));- v[0] = q4[0];- v[1] = q4[4];-- const uint16_t* scales = (const uint16_t*)bq4_K->scales;- uint16_t aux[2];- const int j = bq8_offset / 2;- if (j < 2) {- aux[0] = scales[j + 0] & 0x3f3f;- aux[1] = scales[j + 2] & 0x3f3f;- } else {- aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);- aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);- }- const uint8_t* sc = (const uint8_t*)aux;- const uint8_t* m = sc + 2;-- for (int i = 0; i < QR4_K; ++i) {- const block_q8_1* bq8i = bq8_1 + bq8_offset + i;- d8[i] = __low2float(bq8i->ds);-- const int* q8 = (const int*)bq8i->qs + ((iqs / 2) % 4);- u[2 * i + 0] = q8[0];- u[2 * i + 1] = q8[4];- }-- return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q4_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_ql[mmq_y * (WARP_SIZE_GGUF) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI4_K) + mmq_y / QI4_K];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8];-- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_sc = tile_x_sc;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q4_K(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI4_K; // == 0 if QK_K == 256- const int kqsx = k % QI4_K; // == k if QK_K == 256-- const block_q4_K* bx0 = (const block_q4_K*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;-- if (need_check) {- i = min(i, i_max);- }- const block_q4_K* bxi = bx0 + i * blocks_per_row + kbx;- x_ql[i * (WARP_SIZE_GGUF + 1) + k] = get_int_from_uint8_aligned(bxi->qs, kqsx);- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI4_K; // == 1 if QK_K == 256- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI4_K) {- int i = (i0 + i_offset * QI4_K + k / blocks_per_tile_x_row) % mmq_y;- if (need_check) {- i = min(i, i_max);- }- const block_q4_K* bxi = bx0 + i * blocks_per_row + kbxd;- x_dm[i * (WARP_SIZE_GGUF / QI4_K) + i / QI4_K + kbxd] = bxi->dm;- }--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q4_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / (QI4_K / 8);-- const int* scales = (const int*)bxi->scales;-- const int ksc = k % (WARP_SIZE_GGUF / 8);- // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8- int scales8 = (scales[(ksc % 2) + (ksc != 0)] >> (4 * (ksc & (ksc / 2)))) & 0x0F0F0F0F; // lower 4 bits- scales8 |= (scales[ksc / 2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits-- x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + ksc] = scales8;- }-}--static __device__ __forceinline__ float vec_dot_q4_K_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- (void)x_qh;-- const uint8_t* sc = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 16]) + 2 * ((k % 16) / 8);-- const int index_y = j * WARP_SIZE_GGUF + (QR4_K * k) % WARP_SIZE_GGUF;- return vec_dot_q4_K_q8_1_impl_mmq(- &x_ql[i * (WARP_SIZE_GGUF + 1) + k],- &y_qs[index_y],- sc,- sc + 8,- x_dm[i * (WARP_SIZE_GGUF / QI4_K) + i / QI4_K],- &y_ds[index_y / QI8_1]);-}--static __device__ __forceinline__ float-vec_dot_q5_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q5_K* bq5_K = (const block_q5_K*)vbq;-- int vl[2];- int vh[2];- int u[2 * QR5_K];- float d8[QR5_K];-- const int bq8_offset = QR5_K * ((iqs / 2) / (QI8_1 / 2));- const int* ql = (const int*)(bq5_K->qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4));- const int* qh = (const int*)(bq5_K->qh + 4 * ((iqs / 2) % 4));-- vl[0] = ql[0];- vl[1] = ql[4];-- vh[0] = qh[0] >> bq8_offset;- vh[1] = qh[4] >> bq8_offset;-- const uint16_t* scales = (const uint16_t*)bq5_K->scales;- uint16_t aux[2];- const int j = bq8_offset / 2;- if (j < 2) {- aux[0] = scales[j + 0] & 0x3f3f;- aux[1] = scales[j + 2] & 0x3f3f;- } else {- aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2);- aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2);- }- const uint8_t* sc = (const uint8_t*)aux;- const uint8_t* m = sc + 2;--#pragma unroll- for (int i = 0; i < QR5_K; ++i) {- const block_q8_1* bq8i = bq8_1 + bq8_offset + i;- d8[i] = __low2float(bq8i->ds);-- const int* q8 = (const int*)bq8i->qs + ((iqs / 2) % 4);- u[2 * i + 0] = q8[0];- u[2 * i + 1] = q8[4];- }-- return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q5_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI5_K) + mmq_y / QI5_K];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8];-- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_sc = tile_x_sc;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q5_K(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI5_K; // == 0 if QK_K == 256- const int kqsx = k % QI5_K; // == k if QK_K == 256-- const block_q5_K* bx0 = (const block_q5_K*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_K* bxi = bx0 + i * blocks_per_row + kbx;- const int ky = QR5_K * kqsx;-- const int ql = get_int_from_uint8_aligned(bxi->qs, kqsx);- const int ql0 = (ql >> 0) & 0x0F0F0F0F;- const int ql1 = (ql >> 4) & 0x0F0F0F0F;-- const int qh = get_int_from_uint8_aligned(bxi->qh, kqsx % (QI5_K / 4));- const int qh0 = ((qh >> (2 * (kqsx / (QI5_K / 4)) + 0)) << 4) & 0x10101010;- const int qh1 = ((qh >> (2 * (kqsx / (QI5_K / 4)) + 1)) << 4) & 0x10101010;-- const int kq0 = ky - ky % (QI5_K / 2) + k % (QI5_K / 4) + 0;- const int kq1 = ky - ky % (QI5_K / 2) + k % (QI5_K / 4) + (QI5_K / 4);-- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq0] = ql0 | qh0;- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq1] = ql1 | qh1;- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI5_K; // == 1 if QK_K == 256- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI5_K) {- int i = (i0 + i_offset * QI5_K + k / blocks_per_tile_x_row) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_K* bxi = bx0 + i * blocks_per_row + kbxd;- x_dm[i * (WARP_SIZE_GGUF / QI5_K) + i / QI5_K + kbxd] = bxi->dm;- }--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q5_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / (QI5_K / 8);-- const int* scales = (const int*)bxi->scales;-- const int ksc = k % (WARP_SIZE_GGUF / 8);-- // scale arrangement after the following two lines: sc0,...,sc3, sc4,...,sc7, m0,...,m3, m4,...,m8- int scales8 = (scales[(ksc % 2) + (ksc != 0)] >> (4 * (ksc & (ksc / 2)))) & 0x0F0F0F0F; // lower 4 bits- scales8 |= (scales[ksc / 2] >> (2 * (ksc % 2))) & 0x30303030; // upper 2 bits-- x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + ksc] = scales8;- }-}--static __device__ __forceinline__ float vec_dot_q5_K_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- const uint8_t* sc = ((const uint8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 16]) + 2 * ((k % 16) / 8);-- const int index_x = i * (QR5_K * WARP_SIZE_GGUF + 1) + QR5_K * k;- const int index_y = j * WARP_SIZE_GGUF + (QR5_K * k) % WARP_SIZE_GGUF;- return vec_dot_q5_K_q8_1_impl_mmq(- &x_ql[index_x],- &y_qs[index_y],- sc,- sc + 8,- x_dm[i * (WARP_SIZE_GGUF / QI5_K) + i / QI5_K],- &y_ds[index_y / QI8_1]);-}--static __device__ __forceinline__ float-vec_dot_q6_K_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_q6_K* bq6_K = (const block_q6_K*)vbq;-- const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 4);- const int scale_offset = (QI6_K / 4) * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 8);- const int vh_shift = 2 * ((iqs % (QI6_K / 2)) / (QI6_K / 4));-- const int vl = get_int_from_uint8(bq6_K->ql, iqs);- const int vh = get_int_from_uint8(bq6_K->qh, (QI6_K / 4) * (iqs / (QI6_K / 2)) + iqs % (QI6_K / 4)) >> vh_shift;-- const int8_t* scales = bq6_K->scales + scale_offset;-- int u[QR6_K];- float d8[QR6_K];--#pragma unroll- for (int i = 0; i < QR6_K; ++i) {- u[i] = get_int_from_int8_aligned(bq8_1[bq8_offset + 2 * i].qs, iqs % QI8_1);- d8[i] = __low2float(bq8_1[bq8_offset + 2 * i].ds);- }-- return vec_dot_q6_K_q8_1_impl_mmvq(vl, vh, u, scales, __half2float(bq6_K->d), d8);-}--template <int mmq_y>-static __device__ __forceinline__ void allocate_tiles_q6_K(int** x_ql, half2** x_dm, int** x_qh, int** x_sc) {- __shared__ int tile_x_ql[mmq_y * (2 * WARP_SIZE_GGUF) + mmq_y];- __shared__ half2 tile_x_dm[mmq_y * (WARP_SIZE_GGUF / QI6_K) + mmq_y / QI6_K];- __shared__ int tile_x_sc[mmq_y * (WARP_SIZE_GGUF / 8) + mmq_y / 8];-- *x_ql = tile_x_ql;- *x_dm = tile_x_dm;- *x_sc = tile_x_sc;-}--template <int mmq_y, int nwarps, bool need_check>-static __device__ __forceinline__ void load_tiles_q6_K(- const void* __restrict__ vx,- int* __restrict__ x_ql,- half2* __restrict__ x_dm,- int* __restrict__ x_qh,- int* __restrict__ x_sc,- const int& i_offset,- const int& i_max,- const int& k,- const int& blocks_per_row) {- const int kbx = k / QI6_K; // == 0 if QK_K == 256- const int kqsx = k % QI6_K; // == k if QK_K == 256-- const block_q6_K* bx0 = (const block_q6_K*)vx;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps) {- int i = i0 + i_offset;-- if (need_check) {- i = min(i, i_max);- }-- const block_q6_K* bxi = bx0 + i * blocks_per_row + kbx;- const int ky = QR6_K * kqsx;-- const int ql = get_int_from_uint8(bxi->ql, kqsx);- const int ql0 = (ql >> 0) & 0x0F0F0F0F;- const int ql1 = (ql >> 4) & 0x0F0F0F0F;-- const int qh = get_int_from_uint8(bxi->qh, (QI6_K / 4) * (kqsx / (QI6_K / 2)) + kqsx % (QI6_K / 4));- const int qh0 = ((qh >> (2 * ((kqsx % (QI6_K / 2)) / (QI6_K / 4)))) << 4) & 0x30303030;- const int qh1 = (qh >> (2 * ((kqsx % (QI6_K / 2)) / (QI6_K / 4)))) & 0x30303030;-- const int kq0 = ky - ky % QI6_K + k % (QI6_K / 2) + 0;- const int kq1 = ky - ky % QI6_K + k % (QI6_K / 2) + (QI6_K / 2);-- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq0] = __vsubss4(ql0 | qh0, 0x20202020);- x_ql[i * (2 * WARP_SIZE_GGUF + 1) + kq1] = __vsubss4(ql1 | qh1, 0x20202020);- }-- const int blocks_per_tile_x_row = WARP_SIZE_GGUF / QI6_K; // == 1 if QK_K == 256- const int kbxd = k % blocks_per_tile_x_row; // == 0 if QK_K == 256- float* x_dmf = (float*)x_dm;--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * QI6_K) {- int i = (i0 + i_offset * QI6_K + k / blocks_per_tile_x_row) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q6_K* bxi = bx0 + i * blocks_per_row + kbxd;-- x_dmf[i * (WARP_SIZE_GGUF / QI6_K) + i / QI6_K + kbxd] = __half2float(bxi->d);- }--#pragma unroll- for (int i0 = 0; i0 < mmq_y; i0 += nwarps * 8) {- int i = (i0 + i_offset * 8 + k / (WARP_SIZE_GGUF / 8)) % mmq_y;-- if (need_check) {- i = min(i, i_max);- }-- const block_q6_K* bxi = bx0 + i * blocks_per_row + (k % (WARP_SIZE_GGUF / 8)) / 4;-- x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k % (WARP_SIZE_GGUF / 8)] = get_int_from_int8(bxi->scales, k % (QI6_K / 8));- }-}--static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat(- const int* __restrict__ x_ql,- const half2* __restrict__ x_dm,- const int* __restrict__ x_qh,- const int* __restrict__ x_sc,- const int* __restrict__ y_qs,- const half2* __restrict__ y_ds,- const int& i,- const int& j,- const int& k) {- const float* x_dmf = (const float*)x_dm;- const float* y_df = (const float*)y_ds;-- const int8_t* sc = ((const int8_t*)&x_sc[i * (WARP_SIZE_GGUF / 8) + i / 8 + k / 8]);-- const int index_x = i * (QR6_K * WARP_SIZE_GGUF + 1) + QR6_K * k;- const int index_y = j * WARP_SIZE_GGUF + (QR6_K * k) % WARP_SIZE_GGUF;- return vec_dot_q6_K_q8_1_impl_mmq(- &x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE_GGUF / QI6_K) + i / QI6_K], &y_df[index_y / QI8_1]);-}--static __device__ __forceinline__ float-vec_dot_iq2_xxs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_iq2_xxs* bq2 = (const block_iq2_xxs*)vbq;-- const int ib32 = iqs;- const uint16_t* q2 = bq2->qs + 4 * ib32;- const uint8_t* aux8 = (const uint8_t*)q2;- const int8_t* q8 = bq8_1[ib32].qs;- uint32_t aux32 = q2[2] | (q2[3] << 16);- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint8_t* grid = (const uint8_t*)(iq2xxs_grid + aux8[l]);- const uint8_t signs = ksigns_iq2xs[aux32 & 127];- for (int j = 0; j < 8; ++j) {- sumi += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }- q8 += 8;- aux32 >>= 7;- }- const float d = __half2float(bq2->d) * (0.5f + aux32) * __half2float(bq8_1[ib32].ds.x) * 0.25f;- return d * sumi;-}--static __device__ __forceinline__ float-vec_dot_iq2_xs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {- const block_iq2_xs* bq2 = (const block_iq2_xs*)vbq;-- const int ib32 = iqs;- const uint16_t* q2 = bq2->qs + 4 * ib32;- const int8_t* q8 = bq8_1[ib32].qs;- const uint8_t ls1 = bq2->scales[ib32] & 0xf;- const uint8_t ls2 = bq2->scales[ib32] >> 4;- int sumi1 = 0;- for (int l = 0; l < 2; ++l) {- const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[l] & 511));- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];- for (int j = 0; j < 8; ++j) {- sumi1 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }- q8 += 8;- }- int sumi2 = 0;- for (int l = 2; l < 4; ++l) {- const uint8_t* grid = (const uint8_t*)(iq2xs_grid + (q2[l] & 511));- const uint8_t signs = ksigns_iq2xs[q2[l] >> 9];- for (int j = 0; j < 8; ++j) {- sumi2 += q8[j] * grid[j] * (signs & kmask_iq2xs[j] ? -1 : 1);- }- q8 += 8;- }- const float d = __half2float(bq2->d) * __half2float(bq8_1[ib32].ds.x) * 0.25f;- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);-}--static __device__ __forceinline__ float-vec_dot_iq2_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- const block_iq2_s* bq2 = (const block_iq2_s*)vbq;-- const int ib32 = iqs;- const int8_t* q8 = bq8_1[ib32].qs;- const uint8_t* signs = bq2->qs + QK_K / 8 + 4 * ib32;- const uint8_t ls1 = bq2->scales[ib32] & 0xf;- const uint8_t ls2 = bq2->scales[ib32] >> 4;- int sumi1 = 0;- for (int l = 0; l < 2; ++l) {- const uint32_t* grid =- (const uint32_t*)(iq2s_grid + (bq2->qs[4 * ib32 + l] | ((bq2->qh[ib32] << (8 - 2 * l)) & 0x300)));- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);- sumi1 = __dp4a(grid_l, *((const int*)q8 + 0), sumi1);- sumi1 = __dp4a(grid_h, *((const int*)q8 + 1), sumi1);- q8 += 8;- }- int sumi2 = 0;- for (int l = 2; l < 4; ++l) {- const uint32_t* grid =- (const uint32_t*)(iq2s_grid + (bq2->qs[4 * ib32 + l] | ((bq2->qh[ib32] << (8 - 2 * l)) & 0x300)));- const uint32_t signs0 = __vcmpeq4(((signs[l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- const uint32_t signs1 = __vcmpeq4(((signs[l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid[1] ^ signs1, signs1);- sumi2 = __dp4a(grid_l, *((const int*)q8 + 0), sumi2);- sumi2 = __dp4a(grid_h, *((const int*)q8 + 1), sumi2);- q8 += 8;- }- const float d = __half2float(bq2->d) * __low2float(bq8_1[ib32].ds) * 0.25f;- return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2);-#endif-}--static __device__ __forceinline__ float-vec_dot_iq3_xxs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- const block_iq3_xxs* bq2 = (const block_iq3_xxs*)vbq;-- const int ib32 = iqs;- const uint8_t* q3 = bq2->qs + 8 * ib32;- const uint16_t* gas = (const uint16_t*)(bq2->qs + QK_K / 4) + 2 * ib32;- const int8_t* q8 = bq8_1[ib32].qs;- uint32_t aux32 = gas[0] | (gas[1] << 16);- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint32_t* grid1 = iq3xxs_grid + q3[2 * l + 0];- const uint32_t* grid2 = iq3xxs_grid + q3[2 * l + 1];- const uint32_t* signs = (const uint32_t*)(ksigns64 + (aux32 & 127));- const int grid_l = __vsub4(grid1[0] ^ signs[0], signs[0]);- const int grid_h = __vsub4(grid2[0] ^ signs[1], signs[1]);- sumi = __dp4a(grid_l, *((int*)q8 + 0), sumi);- sumi = __dp4a(grid_h, *((int*)q8 + 1), sumi);- q8 += 8;- aux32 >>= 7;- }- const float d = __half2float(bq2->d) * (0.5f + aux32) * __low2float(bq8_1[ib32].ds) * 0.5f;- return d * sumi;-#endif-}--static __device__ __forceinline__ float-vec_dot_iq3_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- const block_iq3_s* bq2 = (const block_iq3_s*)vbq;-- const int ib32 = iqs;- const uint8_t* qs = bq2->qs + 8 * ib32;- const int8_t* q8 = bq8_1[ib32].qs;- int sumi = 0;- for (int l = 0; l < 4; ++l) {- const uint32_t* grid1 = iq3xs_grid + (qs[2 * l + 0] | ((bq2->qh[ib32] << (8 - 2 * l)) & 256));- const uint32_t* grid2 = iq3xs_grid + (qs[2 * l + 1] | ((bq2->qh[ib32] << (7 - 2 * l)) & 256));- uint32_t signs0 = __vcmpeq4(((bq2->signs[4 * ib32 + l] & 0xf) * 0x01010101) & 0x08040201, 0x08040201);- uint32_t signs1 = __vcmpeq4(((bq2->signs[4 * ib32 + l] >> 4) * 0x01010101) & 0x08040201, 0x08040201);- const int grid_l = __vsub4(grid1[0] ^ signs0, signs0);- const int grid_h = __vsub4(grid2[0] ^ signs1, signs1);- sumi = __dp4a(grid_l, *((int*)q8 + 0), sumi);- sumi = __dp4a(grid_h, *((int*)q8 + 1), sumi);- q8 += 8;- }- const float d = __half2float(bq2->d) * (0.5f + ((bq2->scales[ib32 / 2] >> 4 * (ib32 % 2)) & 0xf)) *- __low2float(bq8_1[ib32].ds) * 0.5f;- return d * sumi;-#endif-}--static __device__ __forceinline__ float-vec_dot_iq1_s_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- const block_iq1_s* bq1 = (const block_iq1_s*)vbq;-- const int qs_packed = get_int_b2(bq1->qs, iqs);- const uint8_t* qs = (const uint8_t*)&qs_packed;-- const int qh = bq1->qh[iqs];-- int sumi = 0;-#pragma unroll- for (int l0 = 0; l0 < 8; l0 += 2) {- const int grid = iq1s_grid_gpu[qs[l0 / 2] | (((qh >> 3 * (l0 / 2)) & 0x07) << 8)];-- const int grid0 = (grid >> 0) & 0x0F0F0F0F;- const int grid1 = (grid >> 4) & 0x0F0F0F0F;-- const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);- const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);-- sumi = __dp4a(grid0, u0, sumi);- sumi = __dp4a(grid1, u1, sumi);- }-- const float d1q = __half2float(bq1->d) * (((qh >> 11) & 0x0E) + 1);- const float delta = -1.0f + IQ1S_DELTA - (qh & 0x8000) * (2.0f * IQ1S_DELTA / 0x8000);- const float2 ds = __half22float2(bq8_1[iqs].ds);- return d1q * (ds.x * sumi + ds.y * delta);-#endif-}--static __device__ __forceinline__ float-vec_dot_iq1_m_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA-- const block_iq1_m* bq1 = (const block_iq1_m*)vbq;-- const int qs_packed = get_int_b4(bq1->qs, iqs);- const uint8_t* qs = (const uint8_t*)&qs_packed;-- int sumi[2] = {0};- float sumf[2] = {0.0f};-#pragma unroll- for (int l0 = 0; l0 < 8; l0 += 2) {- const int qhl = bq1->qh[2 * iqs + l0 / 4] >> (4 * ((l0 / 2) % 2));-- const int grid = iq1s_grid_gpu[qs[l0 / 2] | ((qhl & 0x07) << 8)];-- const int grid0 = (grid >> 0) & 0x0F0F0F0F;- const int grid1 = (grid >> 4) & 0x0F0F0F0F;-- const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0);- const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1);-- sumi[l0 / 4] = __dp4a(grid0, u0, sumi[l0 / 4]);- sumi[l0 / 4] = __dp4a(grid1, u1, sumi[l0 / 4]);-- const float delta = -1.0f + IQ1M_DELTA - (qhl & 0x08) * (2.0f * IQ1M_DELTA / 0x08);- int sumy = 0;- sumy = __dp4a(u0, 0x01010101, sumy);- sumy = __dp4a(u1, 0x01010101, sumy);- sumf[l0 / 4] += delta * sumy;- }-- const uint16_t* sc = (const uint16_t*)bq1->scales;-- iq1m_scale_t scale;- scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00F0) | ((sc[2] >> 4) & 0x0F00) | (sc[3] & 0xF000);- const float d = __half2float(scale.f16) * __low2float(bq8_1[iqs].ds);-- const int tmp = sc[iqs / 2] >> (6 * (iqs % 2));- const int sc0 = 2 * ((tmp >> 0) & 0x07) + 1;- const int sc1 = 2 * ((tmp >> 3) & 0x07) + 1;- return d * ((sumi[0] + sumf[0]) * sc0 + (sumi[1] + sumf[1]) * sc1);-#endif-}--static __device__ __forceinline__ void-get_int_from_table_16(const uint32_t& q4, const uint8_t* values, int& val1, int& val2) {- uint32_t aux32;- const uint8_t* q8 = (const uint8_t*)&aux32;- aux32 = q4 & 0x0f0f0f0f;- uint16_t v1 = values[q8[0]] | (values[q8[1]] << 8);- uint16_t v2 = values[q8[2]] | (values[q8[3]] << 8);- val1 = v1 | (v2 << 16);- aux32 = (q4 >> 4) & 0x0f0f0f0f;- v1 = values[q8[0]] | (values[q8[1]] << 8);- v2 = values[q8[2]] | (values[q8[3]] << 8);- val2 = v1 | (v2 << 16);-}--static __device__ __forceinline__ float-vec_dot_iq4_nl_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA-- const block_iq4_nl* bq = (const block_iq4_nl*)vbq;-- const uint16_t* q4 = (const uint16_t*)bq->qs + 2 * iqs;- const int32_t* q8 = (const int32_t*)bq8_1->qs + iqs;-- const uint8_t* values = (const uint8_t*)kvalues_iq4nl;-- int v1, v2;- int sumi1 = 0, sumi2 = 0;- for (int l = 0; l < VDR_Q4_0_Q8_1_MMVQ; ++l) {- const uint32_t aux = q4[2 * l] | (q4[2 * l + 1] << 16);- get_int_from_table_16(aux, values, v1, v2);- sumi1 = __dp4a(v1, q8[l + 0], sumi1);- sumi2 = __dp4a(v2, q8[l + 4], sumi2);- }- const float d = __half2float(bq->d) * __low2float(bq8_1->ds);- return d * (sumi1 + sumi2);-#endif-}--static __device__ __forceinline__ float-vec_dot_iq4_xs_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1, const int& iqs) {-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 610 || defined USE_ROCM || defined USE_MUSA- const block_iq4_xs* bq4 = (const block_iq4_xs*)vbq;- const uint8_t* values = (const uint8_t*)kvalues_iq4nl;-- // iqs is 0...7- const int ib32 = iqs;- const int32_t* q8 = (const int*)bq8_1[ib32].qs;- const uint32_t* q4 = (const uint32_t*)bq4->qs + 4 * ib32;- const int8_t ls = ((bq4->scales_l[ib32 / 2] >> 4 * (ib32 % 2)) & 0xf) | (((bq4->scales_h >> 2 * ib32) & 3) << 4);- const float d = __half2float(bq4->d) * (ls - 32) * __low2float(bq8_1[ib32].ds);- int v1, v2;- int sumi1 = 0, sumi2 = 0;- for (int j = 0; j < 4; ++j) {- get_int_from_table_16(q4[j], values, v1, v2);- sumi1 = __dp4a(v1, q8[j + 0], sumi1);- sumi2 = __dp4a(v2, q8[j + 4], sumi2);- }- return d * (sumi1 + sumi2);-#endif-}diff --git a/python/freetoken/kernel/ggml.py b/python/freetoken/kernel/ggml.pynew file mode 100644index 00000000..b672d58c--- /dev/null+++ b/python/freetoken/kernel/ggml.py@@ -0,0 +1,155 @@+"""llama.cpp's ggml-cuda kernels (vendored under ``csrc/ggml``) as a tvm-ffi module: prebuilt in the kernel cache, else compiled on first use.++Weights stay in their packed ggml blocks. ``mul_mat`` / ``mul_mat_id`` pick MMVQ or MMQ the way llama.cpp's mul_mat / mul_mat_id do.+Activations are fp32, bf16 or fp16 and are quantized to q8_1 first; results come out in the activation dtype and are accumulated in fp32.+"""++from __future__ import annotations++import functools+import pathlib+from typing import TYPE_CHECKING++import torch++from .utils import _is_rocm, load_aot++if TYPE_CHECKING:+ from tvm_ffi import Module++_ROOT = pathlib.Path(__file__).parent / "csrc" / "ggml"+++def load_module(build_directory: str | None = None) -> Module:+ cuda_dir = _ROOT / "ggml-cuda"+ sources = [+ _ROOT / "freetoken" / "ggml_shim.cu",+ _ROOT / "freetoken" / "ggml_ops.cu",+ *(cuda_dir / f for f in ("mmvq.cu", "mmq.cu", "mmid.cu", "quantize.cu", "convert.cu", "getrows.cu")),+ *sorted((cuda_dir / "template-instances").glob("mmq-instance-*.cu")),+ ]+ if _is_rocm():+ flags = ["-ffast-math", "-DGGML_USE_HIP"]+ else:+ # a multi-arch kernel-cache build is ~6x smaller with compressed SASS+ flags = ["-use_fast_math", "--extended-lambda", "--compress-mode=size"]+ return load_aot(+ "ggml",+ cuda_files=[str(s) for s in sources],+ extra_cuda_cflags=flags,+ extra_include_paths=[str(_ROOT / "include"), str(_ROOT / "src"), str(cuda_dir)],+ build_directory=build_directory,+ )+++def _module() -> Module:+ return load_module()+++def _mmq_tail_bytes(ggml_type: int, k: int) -> int:+ return _module().mmq_tail_bytes(ggml_type, k)+++def _check_mmq_tail(w: torch.Tensor, ggml_type: int, k: int) -> None:+ """MMQ tiles read up to MATRIX_ROW_PADDING values past the last row of ``w``, so its storage must hold them."""+ tail = _mmq_tail_bytes(ggml_type, k)+ if tail == 0 or w.numel() == 0:+ return+ end = w.storage_offset() + sum((n - 1) * s for n, s in zip(w.shape, w.stride())) + 1+ if end + tail > w.untyped_storage().nbytes():+ raise ValueError(f"ggml type {ggml_type} K={k} weight needs {tail} bytes of zero padding after its last row for MMQ")+++def mul_mat(w: torch.Tensor, ggml_type: int, x: torch.Tensor, *, out: torch.Tensor | None = None, xq: torch.Tensor | None = None) -> torch.Tensor:+ """``x[M, K] @ w[N, K]^T`` in ``x``'s dtype with ``w`` packed as ``[N, row_bytes]`` uint8, or written into the ``[M, N]`` view ``out``.++ ``xq`` is ``quantize_q8_1(x)``: the vector kernel reads it instead of quantizing ``x`` again.+ """+ m, k = x.shape+ if out is None:+ out = torch.empty(m, w.shape[0], dtype=x.dtype, device=x.device)+ if not uses_mmvq(ggml_type, m):+ _check_mmq_tail(w, ggml_type, k)+ _module().mul_mat(w, ggml_type, x.contiguous(), out, xq)+ return out+++def quantize_q8_1(x: torch.Tensor) -> torch.Tensor:+ """``x[M, K]`` as the vector kernel's q8_1 blocks, to share among GEMVs that read the same ``x``."""+ m, k = x.shape+ out = torch.empty(m, _q8_1_row_bytes(k), dtype=torch.uint8, device=x.device)+ _module().quantize_q8_1(x.contiguous(), out)+ return out+++def _q8_1_row_bytes(k: int) -> int:+ return _module().q8_1_row_bytes(k)+++def uses_mmvq(ggml_type: int, m: int) -> bool:+ """Whether ``mul_mat`` of ``m`` rows on this GPU runs the vector kernel, the one that reads ``xq``."""+ return _module().uses_mmvq(ggml_type, m)+++def mul_mat_id(w: torch.Tensor, ggml_type: int, x: torch.Tensor, ids: torch.Tensor) -> torch.Tensor:+ """Per-route expert GEMM over ``w[E, N, row_bytes]``: returns ``[T, k, N]`` in ``x``'s dtype; ``x`` is ``[T, K]`` (shared by the k routes) or ``[T, k, K]``."""+ t, top_k = ids.shape+ out = torch.empty(t, top_k, w.shape[1], dtype=x.dtype, device=x.device)+ if t > mmvq_max_tokens(ggml_type):+ _check_mmq_tail(w, ggml_type, x.shape[-1])+ _module().mul_mat_id(w, ggml_type, (x.unsqueeze(1) if x.dim() == 2 else x).contiguous(), ids, out)+ return out+++def mul_mat_id_glu(w: torch.Tensor, ggml_type: int, x: torch.Tensor, ids: torch.Tensor, glu: str, limit: float = 0.0) -> torch.Tensor:+ """``glu(gate, up)`` per route in one pass for small ``T``: ``w[E, 2I, row_bytes]`` holds gate rows then up rows; returns ``[T, k, I]`` in ``x``'s dtype.++ ``glu`` is one of ``swiglu``, ``geglu`` (tanh GELU), ``swiglu_oai`` (alpha 1.702, limit 7) and ``swiglu_clamp`` (alpha 1, clamped at ``limit``).+ """+ t, top_k = ids.shape+ out = torch.empty(t, top_k, w.shape[1] // 2, dtype=x.dtype, device=x.device)+ _module().mul_mat_id_glu(w, ggml_type, x.contiguous(), ids, out, glu, limit)+ return out+++def moe_weighted_sum(y: torch.Tensor, weights: torch.Tensor) -> torch.Tensor:+ """``sum_j weights[t, j] * y[t, j, :]`` over ``y[T, k, N]``, accumulated in fp32 and returned in ``y``'s dtype."""+ out = torch.empty(y.shape[0], y.shape[2], dtype=y.dtype, device=y.device)+ _module().moe_weighted_sum(y.contiguous(), weights.float().contiguous(), out)+ return out+++def mmvq_max_tokens(ggml_type: int) -> int:+ """Largest token count the small-batch MoE kernels take on this GPU."""+ return _module().mmvq_max_tokens(ggml_type)+++def dequantize(w: torch.Tensor, ggml_type: int, k: int, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:+ out = torch.empty(w.numel() // w.shape[-1], k, dtype=dtype, device=w.device)+ _module().dequantize(w, ggml_type, k, out)+ return out+++def get_rows(w: torch.Tensor, ggml_type: int, k: int, ids: torch.Tensor, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:+ out = torch.empty(ids.shape[0], k, dtype=dtype, device=w.device)+ _module().get_rows(w, ggml_type, k, ids, out)+ return out+++__all__ = [+ "dequantize",+ "get_rows",+ "load_module",+ "mmvq_max_tokens",+ "moe_weighted_sum",+ "mul_mat",+ "mul_mat_id",+ "mul_mat_id_glu",+ "quantize_q8_1",+ "uses_mmvq",+]diff --git a/python/freetoken/kernel/gguf.py b/python/freetoken/kernel/gguf.pydeleted file mode 100644index 04a16560..00000000--- a/python/freetoken/kernel/gguf.py+++ /dev/null@@ -1,142 +0,0 @@-"""Borrowed llama.cpp GGUF dequant/GEMM CUDA kernels, JIT-compiled on first use.--The ``.cu``/``.cuh`` under ``csrc/gguf/`` are vendored verbatim from sgl-kernel-(``csrc/quantization/gguf/``), which are themselves ports of llama.cpp. We compile-them through ``torch.utils.cpp_extension.load`` (the same toolchain sglang/vllm use)-into a torch-op module and expose the handful of ops the GGUF path needs. This is a-separate, torch-native extension that sits alongside FreeToken's tvm-ffi kernels.--All ops keep the weight in its native GGUF block layout (packed ``uint8`` rows) and-dequantize *inside* the kernel -- no bf16 copy of the weight is ever materialized.-"""--from __future__ import annotations--import functools-import os-import pathlib-import shutil--import torch--_CSRC = pathlib.Path(__file__).parent / "csrc" / "gguf"---def _host_compiler() -> str | None:- """A host compiler nvcc + libtorch headers accept.-- The system default gcc can be too new for the torch headers (gcc 16 hard-errors),- and on this toolchain even nvcc+gcc-13 trips a non-conformant ``typename- decltype`` in ``List_inl.h`` once ``torch::Tensor`` is instantiated -- but nvcc- with ``clang++`` as host compiles it cleanly. So prefer clang++, then fall back- to an older gcc. Override with ``FREETOKEN_GGUF_HOST_CXX``.- """- override = os.environ.get("FREETOKEN_GGUF_HOST_CXX")- if override:- return override- for cxx in ("clang++", "g++-13", "g++-14", "g++-15"):- if shutil.which(cxx):- return cxx- return None---def _c_compiler_for(cxx: str) -> str:- base = os.path.basename(cxx)- if "clang" in base:- return shutil.which("clang") or "clang"- cc = base.replace("g++", "gcc")- return shutil.which(cc) or cc--def _module():- from torch.utils.cpp_extension import load-- extra_cuda_cflags = ["-O3", "--expt-relaxed-constexpr"]- host_cxx = _host_compiler()- if host_cxx is not None:- # Point both nvcc's host pass (-ccbin) and torch's C++ compile (CXX) at a- # libtorch/nvcc-compatible compiler. Force (not setdefault): the system- # default (CXX unset -> g++) can be a gcc too new for the torch headers.- cxx_path = shutil.which(host_cxx) or host_cxx- extra_cuda_cflags += ["-ccbin", cxx_path]- os.environ["CXX"] = cxx_path- os.environ["CC"] = _c_compiler_for(cxx_path)-- # gguf_kernel.cu carries its own PYBIND11_MODULE (appended at the end), so a- # plain `load` of the single source compiles + binds the ggml_* ops.- return load(- name="freetoken_gguf_kernels",- sources=[str(_CSRC / "gguf_kernel.cu")],- extra_include_paths=[str(_CSRC)],- extra_cuda_cflags=extra_cuda_cflags,- verbose=True,- )---# ---- thin typed wrappers (signatures mirror sgl_kernel.quantization.gguf) -------def ggml_dequantize(- weight: torch.Tensor, quant_type: int, m: int, n: int, dtype: torch.dtype | None = None-) -> torch.Tensor:- """Dequantize a packed GGUF weight ``[m, row_bytes]`` to a dense ``[m, n]`` tensor."""- return _module().ggml_dequantize(weight, quant_type, m, n, dtype)---def ggml_mul_mat_vec_a8(- weight: torch.Tensor, x: torch.Tensor, quant_type: int, row: int-) -> torch.Tensor:- """MMVQ: small-batch GEMV with on-the-fly dequant. ``row`` = output features."""- return _module().ggml_mul_mat_vec_a8(weight, x, quant_type, row)---def ggml_mul_mat_a8(- weight: torch.Tensor, x: torch.Tensor, quant_type: int, row: int-) -> torch.Tensor:- """MMQ: large-batch quantized matmul. ``row`` = output features."""- return _module().ggml_mul_mat_a8(weight, x, quant_type, row)---def ggml_moe_a8(- x: torch.Tensor,- weight: torch.Tensor,- sorted_token_ids: torch.Tensor,- expert_ids: torch.Tensor,- num_tokens_post_padded: torch.Tensor,- quant_type: int,- row: int,- top_k: int,- tokens: int,-) -> torch.Tensor:- """MMQ grouped expert matmul over stacked experts ``weight[E, row, *]``."""- return _module().ggml_moe_a8(- x, weight, sorted_token_ids, expert_ids, num_tokens_post_padded,- quant_type, row, top_k, tokens,- )---def ggml_moe_a8_vec(- x: torch.Tensor,- weight: torch.Tensor,- topk_ids: torch.Tensor,- top_k: int,- quant_type: int,- row: int,- tokens: int,-) -> torch.Tensor:- """MMVQ grouped expert GEMV over stacked experts ``weight[E, row, *]``."""- return _module().ggml_moe_a8_vec(x, weight, topk_ids, top_k, quant_type, row, tokens)---def ggml_moe_get_block_size(quant_type: int) -> int:- return _module().ggml_moe_get_block_size(quant_type)---__all__ = [- "ggml_dequantize",- "ggml_mul_mat_vec_a8",- "ggml_mul_mat_a8",- "ggml_moe_a8",- "ggml_moe_a8_vec",- "ggml_moe_get_block_size",-]diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.pyindex cd7a402a..e38a6cae 100644--- a/python/freetoken/layers/activation.py+++ b/python/freetoken/layers/activation.py@@ -6,10 +6,17 @@ if TYPE_CHECKING:import torch-def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):+def _use_flashinfer(x: torch.Tensor, out: torch.Tensor | None) -> bool:+ """flashinfer's *_and_mul kernels take fp16 / bf16 in and the same dtype out; the Triton kernels take any float dtypes."""+ import torch+from freetoken.kernel.backend import is_flashinfer_installed- if is_flashinfer_installed():+ return x.dtype in (torch.float16, torch.bfloat16) and (out is None or out.dtype == x.dtype) and is_flashinfer_installed()+++def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):+ if _use_flashinfer(x, out):from flashinfer import silu_and_mulelse:from freetoken.kernel.triton.activation import silu_and_mul@@ -18,9 +25,7 @@ def silu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):def gelu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):- from freetoken.kernel.backend import is_flashinfer_installed-- if is_flashinfer_installed():+ if _use_flashinfer(x, out):from flashinfer import gelu_and_mulelse:from freetoken.kernel.triton.activation import gelu_and_mul@@ -30,9 +35,7 @@ def gelu_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):"""tanh-approximate GELU gate (`gelu_pytorch_tanh`) followed by elementwise mul."""- from freetoken.kernel.backend import is_flashinfer_installed-- if is_flashinfer_installed():+ if _use_flashinfer(x, out):from flashinfer import gelu_tanh_and_mulelse:from freetoken.kernel.triton.activation import gelu_tanh_and_muldiff --git a/python/freetoken/layers/embedding.py b/python/freetoken/layers/embedding.pyindex a2221654..f86670b1 100644--- a/python/freetoken/layers/embedding.py+++ b/python/freetoken/layers/embedding.py@@ -3,7 +3,6 @@ from __future__ import annotationsfrom typing import Dictimport torch-import torch.nn.functional as Ffrom freetoken.core import get_global_ctxfrom freetoken.distributed import DistributedCommunicator, get_tp_infofrom freetoken.utils import div_ceil, nvtx_annotate@@ -13,11 +12,18 @@ from .quantization import LayerKind, QuantConfig, quant_method_forclass VocabParallelEmbedding(BaseOP):+ """The vocab table, sharded over TP: its ``quant_method`` owns the table, looks rows up, and serves a tied LM head's product."""++ quant_layer_kind = LayerKind.EMBEDDING+def __init__(self,num_embeddings: int,embedding_dim: int,embed_scale: float | None = None,+ *,+ quant_config: QuantConfig | None = None,+ prefix: str = "",):super().__init__()tp_info = get_tp_info()@@ -28,7 +34,12 @@ class VocabParallelEmbedding(BaseOP):start_idx = self.num_embeddings_tp * tp_rankfinish_idx = min(start_idx + self.num_embeddings_tp, num_embeddings)self.vocab_range = (start_idx, finish_idx - start_idx)- self.weight = torch.empty(self.num_embeddings_tp, embedding_dim)+ self.embedding_dim = embedding_dim+ # the LM head subclass is a linear and builds its own method+ self.quant_method = None+ if self.quant_layer_kind is LayerKind.EMBEDDING:+ self.quant_method = quant_method_for(quant_config, self, prefix)+ self.quant_method.create_weights(self)# Gemma scales embeddings by sqrt(hidden_size). The scale is materialized in# the weight dtype (bf16) to match HF, which downcasts the scalar. The GPU# scalar is built lazily (model __init__ runs on the meta device) and cached@@ -39,14 +50,7 @@ class VocabParallelEmbedding(BaseOP):@nvtx_annotate("Embedding")def forward(self, x: torch.Tensor) -> torch.Tensor:- from freetoken.kernel import indexing-- y = indexing(- weights=self.weight,- indices=x,- vocab_range=self.vocab_range if self.tp_size > 1 else None,- )-+ y = self.quant_method.embed(self, x)if self.tp_size > 1:y = self._comm.all_reduce(y)if self._embed_scale is not None:@@ -83,7 +87,6 @@ class ParallelLMHead(VocabParallelEmbedding):self.in_features = embedding_dimself.out_features = self.num_embeddings_tpself.output_sizes = (self.num_embeddings_tp,)- self.quant_method = Noneif tied_embedding is None:self.quant_method = quant_method_for(quant_config, self, prefix)self.quant_method.create_weights(self)@@ -132,7 +135,7 @@ class ParallelLMHead(VocabParallelEmbedding):del indicesif self.tied_embedding is not None:- logits = F.linear(x, self.tied_embedding.weight, self.bias)+ logits = self.tied_embedding.quant_method.logits(self.tied_embedding, x, self.bias)else:logits = self.quant_method.apply(self, x)if self.tp_size == 1:diff --git a/python/freetoken/layers/gguf.py b/python/freetoken/layers/gguf.pydeleted file mode 100644index ac49b1a5..00000000--- a/python/freetoken/layers/gguf.py+++ /dev/null@@ -1,128 +0,0 @@-"""Native-GGUF quantized layers: weights stay in their packed block layout and are-dequantized *inside* the borrowed llama.cpp kernels (no bf16 copy ever materialized).--Mirrors vLLM/sglang's ``GGUFLinearMethod`` / ``GGUFEmbeddingMethod`` dispatch, ported-onto FreeToken's ``BaseOP``. FreeToken keeps fused projections (qkv, gate_up) as a-single tensor: because Q4_0/K-quants pack each *output row* independently over the-input dim, the loader can concatenate the per-shard packed rows along dim 0 (they-share an input dim, hence the same ``row_bytes``), so a fused layer is still one-``[out, row_bytes]`` qweight -- no per-shard padding bookkeeping needed.--TP is assumed to be 1 (the gemma4 GGUF path restricts to TP=1, like the HF path).-"""--from __future__ import annotations--import torch--from freetoken.models.gguf.dequant import (- BLOCK_SHAPE,- GGML_BF16,- GGML_F16,- GGML_F32,- GGML_NAME,- GGML_Q4_0,- GGML_Q6_K,- GGML_Q8_0,- row_bytes,-)--from .base import BaseOP--# ggml type groups for kernel dispatch (subset we build kernels for).-_UNQUANTIZED = {GGML_F32, GGML_F16, GGML_BF16}-# standard + k-quants: both an MMVQ (small-batch GEMV) and MMQ (large-batch) kernel exist.-_MMVQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K}-_MMQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K}-_DEQUANT = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K}--# Below this token count, the MMVQ GEMV kernel wins (matches vLLM's heuristic).-_MMVQ_SAFE = 6---def fused_mul_mat_gguf(x: torch.Tensor, qweight: torch.Tensor, qweight_type: int) -> torch.Tensor:- """y = x @ dequant(qweight).T, dispatched by batch size and quant type."""- from freetoken.kernel.gguf import (- ggml_dequantize,- ggml_mul_mat_a8,- ggml_mul_mat_vec_a8,- )-- out_features = qweight.shape[0]- if x.shape[0] == 0:- return x.new_empty((0, out_features))- if qweight_type in _UNQUANTIZED:- return x @ qweight.T- if x.shape[0] <= _MMVQ_SAFE and qweight_type in _MMVQ:- return ggml_mul_mat_vec_a8(qweight, x, qweight_type, out_features)- if qweight_type in _MMQ:- return ggml_mul_mat_a8(qweight, x, qweight_type, out_features)- if qweight_type in _DEQUANT:- block, type_size = BLOCK_SHAPE[qweight_type]- in_features = qweight.shape[1] // type_size * block- weight = ggml_dequantize(qweight, qweight_type, out_features, in_features, x.dtype)- return x @ weight.T- raise NotImplementedError(f"unsupported GGUF type {GGML_NAME.get(qweight_type, qweight_type)}")---class GGUFLinear(BaseOP):- """Linear whose weight is a native GGUF block-quantized ``[out, row_bytes]`` tensor."""-- def __init__(- self,- in_features: int,- out_features: int,- quant_type: int,- has_bias: bool = False,- ):- self.in_features = in_features- self.out_features = out_features- self._quant_type = quant_type- self.qweight = torch.empty(out_features, row_bytes(in_features, quant_type), dtype=torch.uint8)- self.bias = torch.empty(out_features) if has_bias else None-- def forward(self, x: torch.Tensor) -> torch.Tensor:- out = fused_mul_mat_gguf(x, self.qweight, self._quant_type)- if self.bias is not None:- out = out + self.bias- return out---class GGUFEmbedding(BaseOP):- """Vocab embedding stored as a native GGUF block-quantized table.-- The full table is never dequantized: only the looked-up rows are gathered (in- packed form) and dequantized per lookup, matching vLLM's ``_apply_gguf_embedding``.- """-- def __init__(- self,- num_embeddings: int,- embedding_dim: int,- quant_type: int,- embed_scale: float | None = None,- ):- self.num_embeddings = num_embeddings- self.embedding_dim = embedding_dim- self._quant_type = quant_type- self.qweight = torch.empty(- num_embeddings, row_bytes(embedding_dim, quant_type), dtype=torch.uint8- )- self._embed_scale = embed_scale- self._embed_scale_t: torch.Tensor | None = None-- def forward(self, x: torch.Tensor) -> torch.Tensor:- from freetoken.kernel.gguf import ggml_dequantize-- flat = x.flatten()- rows = self.qweight.index_select(0, flat) # [n, row_bytes] packed- y = ggml_dequantize(rows, self._quant_type, flat.shape[0], self.embedding_dim, torch.bfloat16)- y = y.view(*x.shape, self.embedding_dim)- if self._embed_scale is not None:- if self._embed_scale_t is None:- self._embed_scale_t = torch.tensor(self._embed_scale, dtype=y.dtype, device=y.device)- y = y * self._embed_scale_t- return y---__all__ = ["GGUFLinear", "GGUFEmbedding", "fused_mul_mat_gguf"]diff --git a/python/freetoken/layers/linear.py b/python/freetoken/layers/linear.pyindex d8a3e861..599ca049 100644--- a/python/freetoken/layers/linear.py+++ b/python/freetoken/layers/linear.py@@ -63,6 +63,7 @@ class LinearReplicated(_LinearTPImpl):output_size: int,has_bias: bool,*,+ output_sizes: List[int] | None = None,quant_config: QuantConfig | None = None,prefix: str = "",):@@ -72,6 +73,7 @@ class LinearReplicated(_LinearTPImpl):local_isize=input_size,local_osize=output_size,has_bias=has_bias,+ output_sizes=output_sizes,quant_config=quant_config,prefix=prefix,)diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.pyindex 79ad9e3a..598bd119 100644--- a/python/freetoken/layers/moe.py+++ b/python/freetoken/layers/moe.py@@ -81,16 +81,12 @@ class MoELayer(BaseOP):self.strategy = strategyself.decode_target = decode_targetself.prefix = prefix- # offload layers without a quant config stay on the format-tag banks (GGUF q4_0)- self.quant_method = None- if quant_config is not None or allocate_experts:- self.quant_method = quant_method_for(quant_config, self, prefix)- if allocate_experts:- self.quant_method.create_weights(self)+ self.quant_method = quant_method_for(quant_config, self, prefix)+ if allocate_experts:+ self.quant_method.create_weights(self)def finalize(self) -> None:- if self.quant_method is not None:- self.quant_method.finalize(self)+ self.quant_method.finalize(self)def _maybe_all_reduce(self, hidden_states: torch.Tensor) -> torch.Tensor:if self.tp_size > 1:@@ -103,7 +99,6 @@ class MoELayer(BaseOP):topk_weights: torch.Tensor,topk_ids: torch.Tensor,) -> torch.Tensor:- assert self.quant_method is not Nonereturn self.quant_method.apply(hidden_states, topk_weights, topk_ids, self.quant_method.resident_view(self),layer=self, is_prefill=get_global_ctx().batch.is_prefill,@@ -191,7 +186,7 @@ class OffloadMoELayer(MoELayer):@propertydef skips_inactive_slots(self) -> bool:- return self.quant_method is not None and self.quant_method.kernel.supports_inactive_slots+ return self.quant_method.kernel.supports_inactive_slotsdef forward(self,@@ -321,7 +316,7 @@ class OffloadMoELayer(MoELayer):assert executor is not None, "CPU MoE executor was not initialized"# the kernel reads every route's slot: give the inactive ones slot 0, which holds a# complete payload or the bank's zero fill (never uninitialized storage), times weight 0- # with several quants this holds only because the cache zeroes staged slot 0 rows (zero_inactive_route_slots)+ # with several quants this holds only because the cache zeroes its staging area after prefillclamp = not self.skips_inactive_slotsgpu_slots, gpu_w, cpu_ids = cache.ensure_experts_hybrid(self.layer_id, topk_ids, topk_weights, clamp=clamp)pending = executor.decode_submit(self.layer_id, hidden_states, topk_weights, cpu_ids)@@ -402,7 +397,6 @@ class OffloadMoELayer(MoELayer):# ------------------------------------------------------------------# Kernel dispatch: ``views`` are the bank tensors the movement step produced (in bank registration order) and ``topk_ids`` already index their rows.- # GGUF q4_0 experts still dispatch on the cache's format tag until they get a method.# ------------------------------------------------------------------def _expert_gemm(@@ -417,27 +411,15 @@ class OffloadMoELayer(MoELayer):alphas: tuple[torch.Tensor, torch.Tensor] | None,is_prefill: bool,) -> torch.Tensor:- if self.quant_method is not None:- from freetoken.moe.legacy_format import canonical_role # legacy_format imports this package+ from freetoken.moe.legacy_format import canonical_role # legacy_format imports this package- view = ExpertView(- {canonical_role(name): t for name, t in zip(cache.bank_schema, views)},- slots=None if n is not None else topk_ids, n=n, alphas=alphas,- )- return self.quant_method.apply(- hidden_states, topk_weights, topk_ids, view, layer=self, is_prefill=is_prefill- )- fmt = cache.quant_format- if fmt == "q4_0":- # Native GGUF Q4_0 experts: dequant-in-kernel grouped GEMV (MMVQ) over the- # streamed packed banks; topk_ids already index the cache slots / layer.- from freetoken.moe.fused_q4_0 import fused_experts_gguf_q4_0-- gate_up, down = views- return fused_experts_gguf_q4_0(- hidden_states, gate_up, down, topk_weights, topk_ids, self.activation- )- raise AssertionError(f"offload experts without a quant method only serve q4_0 banks, got {fmt!r}")+ view = ExpertView(+ {canonical_role(name): t for name, t in zip(cache.bank_schema, views)},+ slots=None if n is not None else topk_ids, n=n, alphas=alphas,+ )+ return self.quant_method.apply(+ hidden_states, topk_weights, topk_ids, view, layer=self, is_prefill=is_prefill+ )def iter_moe_layers(model) -> Iterator[MoELayer]:diff --git a/python/freetoken/layers/quantization/__init__.py b/python/freetoken/layers/quantization/__init__.pyindex 62019fc5..905b80ef 100644--- a/python/freetoken/layers/quantization/__init__.py+++ b/python/freetoken/layers/quantization/__init__.py@@ -1,6 +1,7 @@from .scheme import GroupShape, QuantKind, QuantScheme, WeightDescfrom .registry import LayerKind, dialects, method_class, methods_for, register_dialect, register_methodfrom .method import KernelSelectionError, QuantMethod, finalize_quant, select_kernel+from .embedding import EmbeddingConfig, EmbeddingKernel, EmbeddingMethod, GGUFEmbeddingMethod, UnquantizedEmbeddingMethodfrom .linear import (Fp8BlockLinearMethod,Fp8TensorLinearMethod,@@ -28,6 +29,7 @@ from .names import NameMapfrom .configs import (CompressedTensorsConfig,Fp8BlockConfig,+ GGUFConfig,ModelOptConfig,Mxfp4Config,NoQuantConfig,@@ -42,11 +44,12 @@ __all__ = ["QuantKind", "GroupShape", "QuantScheme", "WeightDesc","LayerKind", "dialects", "method_class", "methods_for", "register_dialect", "register_method","KernelSelectionError", "select_kernel", "QuantMethod",+ "EmbeddingConfig", "EmbeddingKernel", "EmbeddingMethod", "UnquantizedEmbeddingMethod", "GGUFEmbeddingMethod","LinearConfig", "LinearKernel", "LinearMethod","UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod","BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod","UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod","QuantBackend", "set_quant_backend", "get_quant_backend", "NameMap",- "QuantConfig", "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config",+ "QuantConfig", "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "GGUFConfig", "Mxfp4Config","quant_method_for", "quantization_config_of", "set_quant_config", "get_quant_config", "finalize_quant",]diff --git a/python/freetoken/layers/quantization/configs/__init__.py b/python/freetoken/layers/quantization/configs/__init__.pyindex 4bd7a6a9..358e6fec 100644--- a/python/freetoken/layers/quantization/configs/__init__.py+++ b/python/freetoken/layers/quantization/configs/__init__.py@@ -1,11 +1,12 @@"""One QuantConfig per checkpoint dialect; importing the package registers them all."""from .base import NoQuantConfig, QuantConfig, quantization_config_of-from . import compressed_tensors, fp8, modelopt, mxfp4+from . import compressed_tensors, fp8, gguf, modelopt, mxfp4from .compressed_tensors import CompressedTensorsConfigfrom .fp8 import Fp8BlockConfigfrom .modelopt import ModelOptConfigfrom .mxfp4 import Mxfp4Config+from .gguf import GGUFConfig_NO_QUANT = NoQuantConfig()@@ -33,6 +34,6 @@ def get_quant_config() -> QuantConfig | None:__all__ = ["QuantConfig", "quantization_config_of", "quant_method_for", "set_quant_config", "get_quant_config",- "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config",- "compressed_tensors", "fp8", "modelopt", "mxfp4",+ "NoQuantConfig", "ModelOptConfig", "CompressedTensorsConfig", "Fp8BlockConfig", "Mxfp4Config", "GGUFConfig",+ "compressed_tensors", "fp8", "gguf", "modelopt", "mxfp4",]diff --git a/python/freetoken/layers/quantization/configs/base.py b/python/freetoken/layers/quantization/configs/base.pyindex a6413610..78818e80 100644--- a/python/freetoken/layers/quantization/configs/base.py+++ b/python/freetoken/layers/quantization/configs/base.py@@ -6,14 +6,15 @@ from abc import ABC, abstractmethodfrom dataclasses import dataclassfrom typing import Any, ClassVar+from ..embedding import EmbeddingConfigfrom ..linear import LinearConfigfrom ..moe import MoEConfigfrom ..names import Matcher, NameMap, name_setfrom ..quant_backend import get_quant_backend-from ..registry import LayerKind, dialects, method_class, register_dialect+from ..registry import LayerKind, dialects, method_class, methods_for, register_dialectfrom ..scheme import QuantKind, QuantScheme-LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig}+LAYER_CONFIGS = {LayerKind.LINEAR: LinearConfig, LayerKind.MOE: MoEConfig, LayerKind.EMBEDDING: EmbeddingConfig}def cfg_get(cfg: Any, key: str, default: Any = None) -> Any:@@ -88,6 +89,9 @@ class QuantConfig(ABC):scheme = self.scheme_for(prefix)layer_kind = layer.quant_layer_kindkind = scheme.kind if scheme else QuantKind.NONE+ if layer_kind is LayerKind.EMBEDDING and kind not in methods_for(layer_kind):+ # a checkpoint that quantizes its linears stores the vocab table dense+ scheme, kind = None, QuantKind.NONEcls = method_class(kind, layer_kind)cfg = self.layer_config(layer, layer_kind, scheme)return cls(cfg, get_quant_backend().select(layer_kind, kind))diff --git a/python/freetoken/layers/quantization/configs/gguf.py b/python/freetoken/layers/quantization/configs/gguf.pynew file mode 100644index 00000000..bbc24f13--- /dev/null+++ b/python/freetoken/layers/quantization/configs/gguf.py@@ -0,0 +1,82 @@+"""GGUF: every tensor carries its own ggml type, read from the file's tensor table."""++from __future__ import annotations++from typing import Any, Callable, ClassVar++from ..ggml_types import is_quantized, type_name+from ..names import NameMap+from ..quant_backend import get_quant_backend+from ..registry import LayerKind, method_class, register_dialect+from ..scheme import QuantKind, QuantScheme, gguf_scheme+from .base import QuantConfig, Stored+++@register_dialect+class GGUFConfig(QuantConfig):+ """``tensor_types`` is the file's name -> ggml type table; ``to_gguf(name, tensor_types)`` (the family's adapter) maps a checkpoint module name to the GGUF tensors that store it."""++ dialect = "gguf"+ STORAGE: ClassVar[dict[QuantKind, dict[str, str | Stored]]] = {QuantKind.GGUF: {"weight": "weight"}}++ def __init__(+ self,+ tensor_types: dict[str, int],+ to_gguf: Callable[[str, dict[str, int]], tuple[str, ...]],+ name_map: NameMap | None = None,+ unquantized: tuple[str, ...] = (),+ ):+ super().__init__(name_map, unquantized)+ self.tensor_types = tensor_types+ self.to_gguf = to_gguf++ @classmethod+ def claims(cls, q: dict[str, Any]) -> bool:+ return False++ def _part_types(self, name: str) -> tuple[int, ...]:+ types = []+ for tensor in self.to_gguf(name, self.tensor_types):+ key = f"{tensor}.weight" if f"{tensor}.weight" in self.tensor_types else tensor+ if key not in self.tensor_types:+ raise KeyError(f"GGUF has no tensor {key!r} for {name!r}")+ types.append(self.tensor_types[key])+ return tuple(types)++ def scheme_for_name(self, name: str) -> QuantScheme | None:+ types = self._part_types(name)+ if not any(is_quantized(t) for t in types):+ return None+ if len(types) != 1:+ raise NotImplementedError(f"{name} is stored as {len(types)} quantized GGUF tensors")+ return gguf_scheme(types)++ def scheme_for(self, prefix: str) -> QuantScheme | None:+ if prefix in self._schemes:+ return self._schemes[prefix]+ names = self.name_map.to_checkpoint(prefix)+ parts = [() if self.unquantized(n) else self._part_types(n) for n in names]+ quantized = [any(is_quantized(t) for t in p) for p in parts]+ if not any(quantized):+ scheme = None+ elif any(len(p) != 1 for p in parts):+ found = {n: [type_name(t) for t in p] for n, p in zip(names, parts)}+ raise NotImplementedError(f"fused module {prefix!r} needs one GGUF tensor per part, got {found}")+ else:+ scheme = gguf_scheme(p[0] for p in parts)+ self._schemes[prefix] = scheme+ return scheme++ def get_quant_method(self, layer: Any, prefix: str):+ if layer.quant_layer_kind is not LayerKind.MOE:+ return super().get_quant_method(layer, prefix)+ # every expert layer loads its GGUF banks, float experts included; the GGUF MoE kernel picks how to multiply them+ parts = [self._part_types(n) for n in self.name_map.to_checkpoint(prefix)]+ if any(len(p) != 1 for p in parts):+ raise NotImplementedError(f"experts {prefix!r} are stored as {parts} GGUF tensors per projection")+ scheme = gguf_scheme(p[0] for p in parts)+ cfg = self.layer_config(layer, LayerKind.MOE, scheme)+ return method_class(QuantKind.GGUF, LayerKind.MOE)(cfg, get_quant_backend().select(LayerKind.MOE, QuantKind.GGUF))+++__all__ = ["GGUFConfig"]diff --git a/python/freetoken/layers/quantization/embedding/__init__.py b/python/freetoken/layers/quantization/embedding/__init__.pynew file mode 100644index 00000000..ccdbece0--- /dev/null+++ b/python/freetoken/layers/quantization/embedding/__init__.py@@ -0,0 +1,10 @@+from .base import EmbeddingConfig, EmbeddingKernel, EmbeddingMethod+from . import gguf, unquantized+from .gguf import GGUFEmbeddingMethod+from .unquantized import UnquantizedEmbeddingMethod++__all__ = [+ "EmbeddingConfig", "EmbeddingKernel", "EmbeddingMethod",+ "UnquantizedEmbeddingMethod", "GGUFEmbeddingMethod",+ "gguf", "unquantized",+]diff --git a/python/freetoken/layers/quantization/embedding/base.py b/python/freetoken/layers/quantization/embedding/base.pynew file mode 100644index 00000000..caa873f4--- /dev/null+++ b/python/freetoken/layers/quantization/embedding/base.py@@ -0,0 +1,64 @@+"""The embedding side: per-layer config, the kernel ABC and the Method base. The method owns the vocab table: it looks rows up, and a tied LM head multiplies by the same table through it."""++from __future__ import annotations++from abc import ABC, abstractmethod+from dataclasses import dataclass+from typing import Any, ClassVar++import torch++from ..method import QuantMethod+from ..scheme import QuantScheme+++@dataclass(frozen=True)+class EmbeddingConfig:+ """The TP-local table shape, the dtype a lookup returns, and the table's scheme."""++ num_embeddings: int+ embedding_dim: int+ tp_size: int+ dtype: torch.dtype+ scheme: QuantScheme | None = None++ @classmethod+ def from_layer(cls, layer: Any, scheme: QuantScheme | None) -> "EmbeddingConfig":+ return cls(layer.num_embeddings_tp, layer.embedding_dim, layer.tp_size, torch.get_default_dtype(), scheme)+++class EmbeddingKernel(ABC):+ """One backend for one table kind: the row lookup, and ``x @ table^T`` for a tied LM head."""++ name: ClassVar[str]++ def unusable_reason(self, cfg: EmbeddingConfig) -> str | None:+ return None++ def worth_it(self, cfg: EmbeddingConfig) -> bool:+ return True++ def finalize(self, layer: Any) -> None:+ pass++ @abstractmethod+ def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor: ...++ @abstractmethod+ def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor: ...+++class EmbeddingMethod(QuantMethod):+ """Declares the table for its kind; the lookup and the tied head's product go to the kernel."""++ @abstractmethod+ def create_weights(self, layer: Any) -> None: ...++ def finalize(self, layer: Any) -> None:+ self.kernel.finalize(layer)++ def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:+ return self.kernel.embed(layer, ids)++ def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:+ return self.kernel.logits(layer, x, bias)diff --git a/python/freetoken/layers/quantization/embedding/gguf.py b/python/freetoken/layers/quantization/embedding/gguf.pynew file mode 100644index 00000000..f41b7b6a--- /dev/null+++ b/python/freetoken/layers/quantization/embedding/gguf.py@@ -0,0 +1,45 @@+"""GGUF tables: packed ggml rows, decoded per looked-up row; a tied head runs the GGUF linear kernel on the same rows."""++from __future__ import annotations++from types import SimpleNamespace+from typing import Any++import torch++from ..ggml_types import Segment, pad_rows, row_bytes+from ..linear.gguf import GgmlLinearKernel+from ..registry import LayerKind, register_method+from ..scheme import QuantKind, gguf_types+from .base import EmbeddingConfig, EmbeddingKernel, EmbeddingMethod+++class GgmlEmbeddingKernel(EmbeddingKernel):+ name = "ggml"++ def unusable_reason(self, cfg: EmbeddingConfig) -> str | None:+ return "GGUF tables are not sharded over TP" if cfg.tp_size > 1 else None++ def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel import ggml++ cfg = layer.quant_method.cfg+ (ggml_type,) = gguf_types(cfg.scheme)+ rows = ggml.get_rows(layer.weight, ggml_type, cfg.embedding_dim, ids.reshape(-1).to(torch.int32), cfg.dtype)+ return rows.view(*ids.shape, cfg.embedding_dim)++ def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:+ cfg = layer.quant_method.cfg+ (ggml_type,) = gguf_types(cfg.scheme)+ table = SimpleNamespace(_ggml_segments=[Segment(ggml_type, 0, 1, 0, cfg.num_embeddings)], weight=layer.weight, bias=bias)+ return GgmlLinearKernel().apply(table, x)+++@register_method(QuantKind.GGUF, LayerKind.EMBEDDING)+class GGUFEmbeddingMethod(EmbeddingMethod):+ candidates = (GgmlEmbeddingKernel,)++ def create_weights(self, layer: Any) -> None:+ (ggml_type,) = gguf_types(self.cfg.scheme)+ h = self.cfg.embedding_dim+ layer.weight = torch.empty(self.cfg.num_embeddings + pad_rows(h), row_bytes(ggml_type, h), dtype=torch.uint8)diff --git a/python/freetoken/layers/quantization/embedding/unquantized.py b/python/freetoken/layers/quantization/embedding/unquantized.pynew file mode 100644index 00000000..5b755baf--- /dev/null+++ b/python/freetoken/layers/quantization/embedding/unquantized.py@@ -0,0 +1,33 @@+"""Dense tables: the indexing kernel looks rows up; a tied head runs the unquantized linear kernel on the same table."""++from __future__ import annotations++from types import SimpleNamespace+from typing import Any++import torch++from ..linear.unquantized import TorchLinearKernel+from ..registry import LayerKind, register_method+from ..scheme import QuantKind+from .base import EmbeddingKernel, EmbeddingMethod+++class IndexingEmbeddingKernel(EmbeddingKernel):+ name = "indexing"++ def embed(self, layer: Any, ids: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel import indexing++ return indexing(weights=layer.weight, indices=ids, vocab_range=layer.vocab_range if layer.tp_size > 1 else None)++ def logits(self, layer: Any, x: torch.Tensor, bias: torch.Tensor | None) -> torch.Tensor:+ return TorchLinearKernel().apply(SimpleNamespace(weight=layer.weight, bias=bias), x)+++@register_method(QuantKind.NONE, LayerKind.EMBEDDING)+class UnquantizedEmbeddingMethod(EmbeddingMethod):+ candidates = (IndexingEmbeddingKernel,)++ def create_weights(self, layer: Any) -> None:+ layer.weight = torch.empty(self.cfg.num_embeddings, self.cfg.embedding_dim)diff --git a/python/freetoken/layers/quantization/ggml_types.py b/python/freetoken/layers/quantization/ggml_types.pynew file mode 100644index 00000000..8b257cd8--- /dev/null+++ b/python/freetoken/layers/quantization/ggml_types.py@@ -0,0 +1,88 @@+"""ggml type facts shared by the GGUF dialect, its methods and the GGUF weight readers."""++from __future__ import annotations++from dataclasses import dataclass++import torch++from freetoken.third_party import gguf++# ggml reads up to this many values past the end of a packed matrix (llama.cpp MATRIX_ROW_PADDING)+MATRIX_ROW_PADDING = 512++UNQUANTIZED = frozenset(+ int(t) for t in (gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16, gguf.GGMLQuantizationType.BF16)+)+++_FLOAT_DTYPES = {+ int(gguf.GGMLQuantizationType.F32): torch.float32,+ int(gguf.GGMLQuantizationType.F16): torch.float16,+ int(gguf.GGMLQuantizationType.BF16): torch.bfloat16,+}+++def float_dtype(ggml_type: int) -> torch.dtype | None:+ """The torch dtype of an unquantized ggml type, None for a quantized one."""+ return _FLOAT_DTYPES.get(int(ggml_type))+++def type_name(ggml_type: int) -> str:+ return gguf.GGMLQuantizationType(ggml_type).name+++def is_quantized(ggml_type: int) -> bool:+ return int(ggml_type) not in UNQUANTIZED+++def row_bytes(ggml_type: int, k: int) -> int:+ block, size = gguf.GGML_QUANT_SIZES[gguf.GGMLQuantizationType(ggml_type)]+ if k % block:+ raise ValueError(f"{type_name(ggml_type)} rows need a multiple of {block} values, got {k}")+ return k // block * size+++def pad_rows(k: int) -> int:+ """Zero rows to append after a packed [rows, K] matrix so ggml's tail reads stay inside it."""+ return 0 if k % MATRIX_ROW_PADDING == 0 else 1+++def mmq_tail_bytes(ggml_type: int, k: int) -> int:+ """Bytes past the last row of a packed [rows, K] matrix that MMQ reads and that must read as zeros."""+ rem = k % MATRIX_ROW_PADDING+ return 0 if rem == 0 or not is_quantized(ggml_type) else row_bytes(ggml_type, MATRIX_ROW_PADDING - rem)+++@dataclass(frozen=True)+class Segment:+ """A run of consecutive fused parts that share one ggml type: one packed matrix and one GEMM."""++ ggml_type: int+ first_part: int+ num_parts: int+ out_start: int+ rows: int+++def segments(types: tuple[int, ...], output_sizes: tuple[int, ...]) -> list[Segment]:+ if len(types) != len(output_sizes):+ raise ValueError(f"{len(types)} ggml types for {len(output_sizes)} fused parts")+ out: list[Segment] = []+ start = 0+ for i, (t, n) in enumerate(zip(types, output_sizes)):+ if out and out[-1].ggml_type == t:+ last = out[-1]+ out[-1] = Segment(t, last.first_part, last.num_parts + 1, last.out_start, last.rows + n)+ else:+ out.append(Segment(int(t), i, 1, start, n))+ start += n+ return out+++def segment_param(index: int) -> str:+ """State-dict leaf of segment ``index``: ``weight`` for the first, then ``weight_1``, ``weight_2``."""+ return "weight" if index == 0 else f"weight_{index}"+++__all__ = ["MATRIX_ROW_PADDING", "Segment", "float_dtype", "is_quantized", "mmq_tail_bytes", "pad_rows", "row_bytes", "segment_param", "segments", "type_name"]diff --git a/python/freetoken/layers/quantization/linear/__init__.py b/python/freetoken/layers/quantization/linear/__init__.pyindex 8236abe0..781f2b45 100644--- a/python/freetoken/layers/quantization/linear/__init__.py+++ b/python/freetoken/layers/quantization/linear/__init__.py@@ -1,13 +1,14 @@from .base import LinearConfig, LinearKernel, LinearMethod-from . import fp8_block, fp8_tensor, mxfp8, nvfp4, unquantized+from . import fp8_block, fp8_tensor, gguf, mxfp8, nvfp4, unquantizedfrom .fp8_block import Fp8BlockLinearMethodfrom .fp8_tensor import Fp8TensorLinearMethodfrom .mxfp8 import Mxfp8LinearMethodfrom .nvfp4 import Nvfp4LinearMethodfrom .unquantized import UnquantizedLinearMethod+from .gguf import GGUFLinearMethod__all__ = ["LinearConfig", "LinearKernel", "LinearMethod",- "UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod",- "fp8_block", "fp8_tensor", "mxfp8", "nvfp4", "unquantized",+ "UnquantizedLinearMethod", "Fp8TensorLinearMethod", "Fp8BlockLinearMethod", "Mxfp8LinearMethod", "Nvfp4LinearMethod", "GGUFLinearMethod",+ "fp8_block", "fp8_tensor", "gguf", "mxfp8", "nvfp4", "unquantized",]diff --git a/python/freetoken/layers/quantization/linear/gguf.py b/python/freetoken/layers/quantization/linear/gguf.pynew file mode 100644index 00000000..ac08dbe2--- /dev/null+++ b/python/freetoken/layers/quantization/linear/gguf.py@@ -0,0 +1,56 @@+"""GGUF Linear: packed ggml rows multiplied by llama.cpp's MMVQ / MMQ kernels; a fused module runs one GEMM per ggml type, float parts as plain bf16 rows."""++from __future__ import annotations++from typing import Any++import torch++from ..ggml_types import is_quantized, pad_rows, row_bytes, segment_param, segments+from ..registry import LayerKind, register_method+from ..scheme import QuantKind, gguf_types+from .base import LinearConfig, LinearKernel, LinearMethod++++class GgmlLinearKernel(LinearKernel):+ name = "ggml"++ def unusable_reason(self, cfg: LinearConfig) -> str | None:+ return None++ def apply(self, layer: Any, x: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel import ggml++ x2 = x.reshape(-1, x.shape[-1])+ segs = layer._ggml_segments+ m = x2.shape[0]+ y = torch.empty(m, sum(s.rows for s in segs), dtype=x.dtype, device=x.device)+ # every segment reads the same x: quantize it once for all the vector-kernel segments+ xq = ggml.quantize_q8_1(x2) if sum(is_quantized(s.ggml_type) and ggml.uses_mmvq(s.ggml_type, m) for s in segs) > 1 else None+ for i, s in enumerate(segs):+ w = getattr(layer, segment_param(i))[: s.rows]+ out = y[:, s.out_start : s.out_start + s.rows]+ if is_quantized(s.ggml_type):+ ggml.mul_mat(w, s.ggml_type, x2, out=out, xq=xq)+ else:+ torch.mm(x2, w.t(), out=out)+ if layer.bias is not None:+ y = y + layer.bias+ return y.reshape(*x.shape[:-1], y.shape[-1])+++@register_method(QuantKind.GGUF, LayerKind.LINEAR)+class GGUFLinearMethod(LinearMethod):+ candidates = (GgmlLinearKernel,)++ def create_weights(self, layer: Any) -> None:+ g = self.cfg+ segs = segments(gguf_types(g.scheme), g.output_sizes)+ layer._ggml_segments = segs+ k = g.in_features+ for i, s in enumerate(segs):+ if is_quantized(s.ggml_type):+ setattr(layer, segment_param(i), torch.empty(s.rows + pad_rows(k), row_bytes(s.ggml_type, k), dtype=torch.uint8))+ else:+ setattr(layer, segment_param(i), torch.empty(s.rows, k))diff --git a/python/freetoken/layers/quantization/moe/__init__.py b/python/freetoken/layers/quantization/moe/__init__.pyindex a5f2dc66..1230f36f 100644--- a/python/freetoken/layers/quantization/moe/__init__.py+++ b/python/freetoken/layers/quantization/moe/__init__.py@@ -1,13 +1,14 @@from .base import BankSpec, ExpertView, MoEConfig, MoEKernel, MoEMethod-from . import fp8_block, mxfp4, mxfp8, nvfp4, unquantized+from . import fp8_block, gguf, mxfp4, mxfp8, nvfp4, unquantizedfrom .fp8_block import Fp8BlockMoEMethodfrom .mxfp4 import Mxfp4MoEMethodfrom .mxfp8 import Mxfp8MoEMethodfrom .nvfp4 import Nvfp4MoEMethodfrom .unquantized import UnquantizedMoEMethod+from .gguf import GGUFMoEMethod__all__ = ["BankSpec", "ExpertView", "MoEConfig", "MoEKernel", "MoEMethod",- "UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod",- "fp8_block", "mxfp4", "mxfp8", "nvfp4", "unquantized",+ "UnquantizedMoEMethod", "Fp8BlockMoEMethod", "Nvfp4MoEMethod", "Mxfp4MoEMethod", "Mxfp8MoEMethod", "GGUFMoEMethod",+ "fp8_block", "gguf", "mxfp4", "mxfp8", "nvfp4", "unquantized",]diff --git a/python/freetoken/layers/quantization/moe/gguf.py b/python/freetoken/layers/quantization/moe/gguf.pynew file mode 100644index 00000000..78e051da--- /dev/null+++ b/python/freetoken/layers/quantization/moe/gguf.py@@ -0,0 +1,118 @@+"""GGUF experts: packed ggml rows in two banks (gate|up and down), multiplied by llama.cpp's mul_mat_id kernels; float experts run on the bf16 fused MoE."""++from __future__ import annotations++import math++import torch++from ..ggml_types import float_dtype, is_quantized, row_bytes, type_name+from ..registry import LayerKind, register_method+from ..scheme import QuantKind, gguf_types+from .base import BankSpec, ExpertView, MoEConfig, MoEKernel, MoEMethod, gated_epilogue_reason, is_resident, limit_or_inf++++def fused_glu(layer) -> tuple[str, float] | None:+ """The MMVQ epilogue (``ggml.mul_mat_id_glu``) that computes this layer's gated activation, or None to run it as a separate kernel."""+ act, alpha, limit = layer.activation, float(layer.alpha), limit_or_inf(layer)+ if act == "silu" and alpha == 1.0 and limit == math.inf:+ return "swiglu", 0.0+ if act == "gelu_tanh":+ return "geglu", 0.0+ if act == "swiglu_clamp" and alpha == 1.0 and limit != math.inf:+ return "swiglu_clamp", limit+ # ggml's fused swiglu_oai keeps its default alpha and limit+ if act == "swigluoai" and math.isclose(alpha, 1.702, rel_tol=1e-6) and math.isclose(limit, 7.0, rel_tol=1e-6):+ return "swiglu_oai", 0.0+ return None+++def expert_types(cfg: MoEConfig) -> tuple[int, int]:+ gate, up, down = gguf_types(cfg.scheme)+ if gate != up:+ raise NotImplementedError(f"GGUF experts with gate {type_name(gate)} and up {type_name(up)} need one gate_up bank type")+ if is_quantized(gate) != is_quantized(down):+ raise NotImplementedError(f"GGUF experts with gate_up {type_name(gate)} and down {type_name(down)} mix float and quantized rows")+ return gate, down++++class GgmlMoEKernel(MoEKernel):+ name = "ggml"+ cpu_format = "gguf"++ def unusable_reason(self, cfg: MoEConfig) -> str | None:+ if not is_quantized(expert_types(cfg)[0]):+ reason = gated_epilogue_reason(cfg)+ if reason:+ return f"float GGUF experts run on the bf16 fused MoE: {reason}"+ return self._common_reject(cfg, tp_ok=False, cpu_ok=True, plain_silu_only=False)++ def layout(self, cfg: MoEConfig) -> dict[str, BankSpec]:+ gate_up, down = expert_types(cfg)+ i, h = cfg.local_intermediate, cfg.hidden+ dtype = float_dtype(gate_up)+ if dtype is not None:+ return {"gate_up": BankSpec((2 * i, h), dtype), "down": BankSpec((h, i), dtype)}+ return {+ "gate_up": BankSpec((2 * i, row_bytes(gate_up, h)), torch.uint8),+ "down": BankSpec((h, row_bytes(down, i)), torch.uint8),+ }++ def pack(self, pieces, cfg: MoEConfig, out):+ i = cfg.local_intermediate+ out["gate_up"][:, :i].copy_(pieces["gate"])+ out["gate_up"][:, i:].copy_(pieces["up"])+ out["down"].copy_(pieces["down"])+ return {}++ def apply(self, layer, x, topk_weights, topk_ids, view: ExpertView, *, is_prefill: bool):+ from freetoken.kernel import ggml+ from freetoken.layers.activation import gated_act_and_mul++ gate_up_type, down_type = expert_types(layer.quant_method.cfg)+ if not is_quantized(gate_up_type):+ return self._apply_float(layer, x, topk_weights, topk_ids, view, gate_up_type, is_prefill=is_prefill)+ assert not layer.apply_router_weight_on_input, "GGUF experts apply the router weights on the output"+ ids = topk_ids.to(torch.int32)+ gate_up = view.tensors["gate_up"]+ glu = fused_glu(layer)+ small = x.shape[0] <= min(ggml.mmvq_max_tokens(gate_up_type), ggml.mmvq_max_tokens(down_type))+ if glu is not None and small:+ inter = ggml.mul_mat_id_glu(gate_up, gate_up_type, x, ids, *glu)+ else:+ h = ggml.mul_mat_id(gate_up, gate_up_type, x, ids)+ t, k, two_i = h.shape+ inter = torch.empty(t, k, two_i // 2, dtype=h.dtype, device=h.device)+ gated_act_and_mul(layer.activation, h.view(-1, two_i), inter.view(-1, two_i // 2), alpha=float(layer.alpha), limit=limit_or_inf(layer))+ del h # free the gate_up output before the down GEMM allocates its own+ y = ggml.mul_mat_id(view.tensors["down"], down_type, inter, ids)+ return ggml.moe_weighted_sum(y, topk_weights)++ @staticmethod+ def _apply_float(layer, x, topk_weights, topk_ids, view: ExpertView, ggml_type: int, *, is_prefill: bool):+ """Float experts: their banks in their own dtype, through the bf16 fused MoE (the unquantized kernel's path)."""+ from freetoken.moe.fused import fused_experts_decode_impl, fused_experts_impl++ gate_up, down = view.tensors["gate_up"], view.tensors["down"]+ if gate_up.dtype != x.dtype:+ raise NotImplementedError(f"GGUF {type_name(ggml_type)} experts need {gate_up.dtype} activations, got {x.dtype}")+ impl = fused_experts_impl if is_prefill or is_resident(layer) else fused_experts_decode_impl+ return impl(x, gate_up, down, topk_weights, topk_ids, layer.activation, layer.apply_router_weight_on_input, float(layer.alpha), limit_or_inf(layer))+++@register_method(QuantKind.GGUF, LayerKind.MOE)+class GGUFMoEMethod(MoEMethod):+ candidates = (GgmlMoEKernel,)++ @property+ def cpu_format(self) -> str | None:+ # the CPU executor reads Q4_0 experts in their GGUF rows and BF16 experts as its bf16 banks; other ggml types have no CPU kernel yet+ gate_up, down = expert_types(self.cfg)+ names = {type_name(gate_up), type_name(down)}+ if names == {"Q4_0"}:+ return "q4_0"+ if names == {"BF16"}:+ return "bf16"+ return self.kernel.cpu_formatdiff --git a/python/freetoken/layers/quantization/registry.py b/python/freetoken/layers/quantization/registry.pyindex 2d13cf50..c561e55e 100644--- a/python/freetoken/layers/quantization/registry.py+++ b/python/freetoken/layers/quantization/registry.py@@ -11,6 +11,7 @@ from .scheme import QuantKindclass LayerKind(Enum):LINEAR = "linear"MOE = "moe"+ EMBEDDING = "embedding"def __str__(self) -> str:return self.valuediff --git a/python/freetoken/layers/quantization/scheme.py b/python/freetoken/layers/quantization/scheme.pyindex 89f968af..ab43814e 100644--- a/python/freetoken/layers/quantization/scheme.py+++ b/python/freetoken/layers/quantization/scheme.py@@ -20,6 +20,7 @@ class QuantKind(Enum):MXFP8 = "mxfp8"NVFP4 = "nvfp4"MXFP4 = "mxfp4"+ GGUF = "gguf"def __str__(self) -> str:return self.value@@ -105,3 +106,13 @@ def nvfp4_scheme(*, input_scale: bool) -> QuantScheme:def mxfp4_scheme() -> QuantScheme:return QuantScheme(QuantKind.MXFP4, WeightDesc("e2m1", (1, MX_GROUP), "e8m0"), {"weight", "weight_scale"})+++def gguf_scheme(types: Iterable[int]) -> QuantScheme:+ """Packed ggml blocks, one ggml type per fused part (``types`` follows the parts' order)."""+ return QuantScheme(QuantKind.GGUF, WeightDesc("ggml:" + ",".join(str(int(t)) for t in types), None, None), {"weight"})+++def gguf_types(scheme: QuantScheme) -> tuple[int, ...]:+ assert scheme.kind is QuantKind.GGUF, scheme+ return tuple(int(t) for t in scheme.weight.elem.removeprefix("ggml:").split(","))diff --git a/python/freetoken/models/gemma4/__init__.py b/python/freetoken/models/gemma4/__init__.pyindex 9f41f080..4fcf15a3 100644--- a/python/freetoken/models/gemma4/__init__.py+++ b/python/freetoken/models/gemma4/__init__.py@@ -1,11 +1,6 @@from .attention import Gemma4Attentionfrom .config import parse_config-from .gguf import (- dummy_q4_0_expert_sources,- iter_gguf_weights,- load_q4_0_expert_sources,- parse_gguf_config,-)+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguffrom .model import Gemma4ForCausalLM, Gemma4ForConditionalGeneration, Gemma4UnifiedForConditionalGenerationfrom .moe import Gemma4Routerfrom .vision import Gemma4MultimodalEmbedder, Gemma4UnifiedVisionEmbedder, Gemma4VisionModel@@ -32,6 +27,5 @@ __all__ = ["iter_weights_parallel","parse_gguf_config","iter_gguf_weights",- "load_q4_0_expert_sources",- "dummy_q4_0_expert_sources",+ "to_gguf",]diff --git a/python/freetoken/models/gemma4/gguf.py b/python/freetoken/models/gemma4/gguf.pyindex 437822b5..95f89c3c 100644--- a/python/freetoken/models/gemma4/gguf.py+++ b/python/freetoken/models/gemma4/gguf.py@@ -6,12 +6,13 @@ final-logit softcap), so this produces the *same* ``ModelConfig`` as``gemma4.config.parse_config`` -- only the source is GGUF KV metadata instead of aHF config object. transformers' own GGUF->config conversion is rejected by thegemma4 strict dataclass (per-layer ``num_key_value_heads`` array), so we read the-metadata directly. ``expert_quant`` is set to ``"q4_0"`` to route the routed experts-through the native-Q4_0 offload-cache path.+metadata directly. Every tensor keeps its packed ggml type (``QuantKind.GGUF``); the+routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""from __future__ import annotations+import refrom typing import TYPE_CHECKING, Iteratorimport torch@@ -22,7 +23,8 @@ from freetoken.models.config import (RotaryConfig,SWAAttentionGroupConfig,)-from freetoken.models.gguf.dequant import GGML_Q4_0, GGML_Q6_K, dequantize, row_bytes+from freetoken.models.gguf.reader import gguf_tensor_index+from freetoken.models.gguf.tensors import float_tensor, fused, matrixif TYPE_CHECKING:from freetoken.models.gguf.config import GgufConfigShim@@ -36,12 +38,9 @@ def _full_rotary_dim(shim: "GgufConfigShim", full_head_dim: int) -> int:which drives the unrotated tail's frequencies to zero (ggml ops.cpp ``theta / ff``).Count the untouched entries to get the rotated width back."""- from freetoken.models.gguf.reader import iter_gguf_tensors-- for t in iter_gguf_tensors(shim.model_path):- if t.name == "rope_freqs.weight": # F32, so the packed bytes ARE the values- freqs = t.packed().reshape(-1).view(torch.float32)- return int((freqs == 1.0).sum().item()) * 2+ info = gguf_tensor_index(shim.model_path).get("rope_freqs.weight")+ if info is not None:+ return int((float_tensor(info) == 1.0).sum().item()) * 2# A metadata-only GGUF (an FTW dir's source_metadata.gguf) has no tensor table. Every# released gemma4 sets partial_rotary_factor 0.25 on the full layers, and the converter# asserts rope_type == "proportional" (llama.cpp conversion/gemma.py).@@ -112,8 +111,7 @@ def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:model_type="gemma4",architectures=list(shim.architectures),moe_enabled=True,- expert_quant="q4_0",- moe_weight_format="q4_0",+ expert_quant="gguf",use_qk_norm=True,attn_sm_scale=1.0,final_logit_softcapping=float(g("final_logit_softcapping")),@@ -144,8 +142,7 @@ def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:# Weight loading: GGUF tensor names -> FreeToken gemma4 module params.# ---------------------------------------------------------------------------------------# Per-layer 1:1 norm/scale tensors (gguf suffix -> freetoken module-relative name). All-# are tiny F32 tensors dequantized to bf16.+# Per-layer 1:1 norm/scale tensors (gguf suffix -> freetoken module-relative name), all F32._LAYER_SCALAR_MAP = {"attn_norm.weight": "input_layernorm.weight","attn_q_norm.weight": "self_attn.q_norm.weight",@@ -161,14 +158,39 @@ _LAYER_SCALAR_MAP = {"ffn_gate_inp.scale": "feed_forward.router.scale","ffn_down_exps.scale": "feed_forward.router.per_expert_scale",}-# Expert weight tensors (handled by the offload bank loader, not iter_gguf_weights).-_EXPERT_SUFFIXES = ("ffn_gate_up_exps.weight", "ffn_down_exps.weight")--def _to_bf16(t) -> torch.Tensor:- """Dequantize a GgufTensor (F32/F16/Q*) to a dense bf16 tensor of its torch shape."""- flat = dequantize(t.packed().reshape(-1), t.ggml_type, torch.bfloat16)- return flat.reshape(t.shape)+# checkpoint module leaf (gemma4 HF naming) -> GGUF tensor+_LAYER_LEAVES = {+ "self_attn.q_proj": "attn_q",+ "self_attn.k_proj": "attn_k",+ "self_attn.o_proj": "attn_output",+ "mlp.gate_proj": "ffn_gate",+ "mlp.up_proj": "ffn_up",+ "mlp.down_proj": "ffn_down",+ "experts.0.gate_proj": "ffn_gate_exps",+ "experts.0.up_proj": "ffn_up_exps",+ "experts.0.down_proj": "ffn_down_exps",+}+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")+++def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""+ if name == "model.embed_tokens":+ return ("token_embd",)+ if name == "lm_head":+ return ("output",) if "output.weight" in tensors else ()+ m = _LAYER_RE.match(name)+ if m is None:+ return ()+ blk, leaf = f"blk.{m.group(1)}", m.group(2)+ if leaf == "self_attn.v_proj":+ # the full-attention layers store no attn_v: k is reused as v+ return (f"{blk}.attn_v",) if f"{blk}.attn_v.weight" in tensors else (f"{blk}.attn_k",)+ if leaf in ("experts.0.gate_proj", "experts.0.up_proj") and f"{blk}.ffn_gate_up_exps.weight" in tensors:+ return (f"{blk}.ffn_gate_up_exps",)+ tensor = _LAYER_LEAVES.get(leaf)+ return (f"{blk}.{tensor}",) if tensor is not None else ()def _require_tp1(what: str) -> None:@@ -191,284 +213,43 @@ def iter_gguf_weights(include_moe_experts: bool,include_non_moe: bool,) -> Iterator[tuple[str, torch.Tensor]]:- """Yield (param_name, tensor) for every non-expert gemma4 param.-- Quantized projections (attention q/k/v/o, shared-MLP gate/up/down) and the- embedding stay in their native packed block layout and are yielded as ``.qweight``- (uint8); norms / router / per-layer scalars dequantize to bf16. q/k/v and gate/up- are fused by concatenating packed rows along the output dim (same input dim ->- same ``row_bytes``). Routed experts are served from the offload cache, so they are- skipped here (asserts the offload contract like the other MoE models).- """- from freetoken.models.gguf.reader import iter_gguf_tensors+ """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""+ if not include_non_moe:+ returnfrom freetoken.utils import cached_load_hf_config- assert not include_moe_experts, (- "gemma4 GGUF stores experts as Q4_0 and only supports the offload backend; "- "experts are loaded into the offload cache via load_q4_0_expert_sources()."- )- assert include_non_moe_require_tp1("weight loading")-- # Full-attention layers ship no attn_v (k reused as v); SWA layers do. Knowing which- # is which lets us emit the fused qkv as soon as its parts are present.config = parse_gguf_config(cached_load_hf_config(model_path))- k_eq_v_layers = {- lid- for lid in range(config.num_layers)- if isinstance(config.attention_group_for_layer(lid), FullAttentionGroupConfig)- and config.attention_group_for_layer(lid).k_eq_v- }-- # Per-layer fusion buffers: layer -> {slot: packed[out, row_bytes]}.- qkv_buf: dict[int, dict[str, torch.Tensor]] = {}- gate_up_buf: dict[int, dict[str, torch.Tensor]] = {}-- def layer_of(name: str) -> int:- return int(name.split(".")[1])-- for t in iter_gguf_tensors(model_path):- name = t.name- if name == "token_embd.weight":- yield "model.embed_tokens.qweight", t.packed() # Q6_K packed table- continue- if name == "output_norm.weight":- yield "model.norm.weight", _to_bf16(t)- continue- if name == "rope_freqs.weight":- continue # rope frequencies recomputed in-engine- if not name.startswith("blk."):- continue- if any(name.endswith(sfx) for sfx in _EXPERT_SUFFIXES):- continue # routed experts -> offload banks-- layer = layer_of(name)- suffix = name.split(".", 2)[2] # after "blk.N."- base = f"model.layers.{layer}"-- if suffix in _LAYER_SCALAR_MAP:- rel = _LAYER_SCALAR_MAP[suffix]- tensor = _to_bf16(t)- if rel.endswith("layer_scalar"):- tensor = tensor.reshape(1)- yield f"{base}.{rel}", tensor- continue-- # Quantized projections: keep packed; fuse q/k/v and gate/up.- if suffix == "attn_q.weight":- qkv_buf.setdefault(layer, {})["q"] = t.packed()- elif suffix == "attn_k.weight":- qkv_buf.setdefault(layer, {})["k"] = t.packed()- elif suffix == "attn_v.weight":- qkv_buf.setdefault(layer, {})["v"] = t.packed()- elif suffix == "ffn_gate.weight":- gate_up_buf.setdefault(layer, {})["gate"] = t.packed()- elif suffix == "ffn_up.weight":- gate_up_buf.setdefault(layer, {})["up"] = t.packed()- elif suffix == "attn_output.weight":- yield f"{base}.self_attn.o_proj.qweight", t.packed()- elif suffix == "ffn_down.weight":- yield f"{base}.feed_forward.shared_mlp.down_proj.qweight", t.packed()- else:- raise ValueError(f"unmapped gemma4 GGUF tensor: {name}")-- # Emit fused qkv once all parts are present: k_eq_v full layers reuse k as v- # (no attn_v tensor), SWA layers wait for their separate attn_v.- slots = qkv_buf.get(layer)- if slots is not None and "q" in slots and "k" in slots:- if layer in k_eq_v_layers:- v = slots["k"]- elif "v" in slots:- v = slots["v"]- else:- v = None- if v is not None:- yield f"{base}.self_attn.qkv_proj.qweight", torch.cat(- [slots["q"], slots["k"], v], dim=0- )- del qkv_buf[layer]- gu = gate_up_buf.get(layer)- if gu is not None and "gate" in gu and "up" in gu:- yield f"{base}.feed_forward.shared_mlp.gate_up_proj.qweight", torch.cat(- [gu["gate"], gu["up"]], dim=0- )- del gate_up_buf[layer]-- assert not qkv_buf, f"incomplete qkv groups: {sorted(qkv_buf)}"- assert not gate_up_buf, f"incomplete gate_up groups: {sorted(gate_up_buf)}"---# ---------------------------------------------------------------------------------------# Model layer swap: dense bf16 Linear/Embedding -> native GGUF-quant ops.-# -----------------------------------------------------------------------------------------def is_gguf_model(config: ModelConfig) -> bool:- """True when the model was parsed from a GGUF checkpoint (native-quant path)."""- return getattr(config, "moe_weight_format", None) == "q4_0"---class GGUFTiedLMHead:- """Tied LM head over a native Q6_K embedding table (logits via ggml matmul).-- Holds only a reference to the GGUF embedding (no params of its own -> empty- state_dict), mirroring ``ParallelLMHead`` with ``tie_word_embeddings``. TP=1 only.- """-- def __init__(self, embedding, quant_type: int):- self._embedding = embedding- self._quant_type = quant_type-- def state_dict(self, *, prefix: str = "", result=None):- return result if result is not None else {}-- def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False):- state_dict.pop(f"{prefix}.weight", None)- state_dict.pop(f"{prefix}.bias", None)-- def forward(self, x: torch.Tensor) -> torch.Tensor:- from freetoken.core import get_global_ctx- from freetoken.layers.gguf import fused_mul_mat_gguf-- batch = get_global_ctx().batch- if batch.is_prefill:- indices = batch.attn_metadata.get_last_indices(batch.size)- x = x[indices].contiguous()- return fused_mul_mat_gguf(x, self._embedding.qweight, self._quant_type)---def convert_gemma4_to_gguf(model, config: ModelConfig) -> None:- """In place: replace gemma4's dense projections + embedding with native GGUF ops.-- Quantized in the checkpoint -> swapped: attention qkv/o, shared-MLP gate_up/down- (all Q4_0) and the token embedding (Q6_K, also the tied LM head). Left as dense- bf16 (F32 in the GGUF): the router gate, all RMSNorms, the per-layer scalars, and- the routed experts (served from the offload cache).- """- from freetoken.layers.gguf import GGUFEmbedding, GGUFLinear-- def swap_linear(owner, attr, quant_type=GGML_Q4_0):- lin = getattr(owner, attr)- out_features, in_features = lin.weight.shape- setattr(- owner,- attr,- GGUFLinear(in_features, out_features, quant_type, has_bias=lin.bias is not None),- )-- inner = model.model- embed = GGUFEmbedding(- num_embeddings=config.vocab_size,- embedding_dim=config.hidden_size,- quant_type=GGML_Q6_K,- embed_scale=config.embedding_scale,- )- inner.embed_tokens = embed-- for layer in inner.layers.op_list:- swap_linear(layer.self_attn, "qkv_proj")- swap_linear(layer.self_attn, "o_proj")- swap_linear(layer.feed_forward.shared_mlp, "gate_up_proj")- swap_linear(layer.feed_forward.shared_mlp, "down_proj")-- if config.tie_word_embeddings:- model.lm_head = GGUFTiedLMHead(embed, GGML_Q6_K)---# ---------------------------------------------------------------------------------------# Routed-expert host banks (native Q4_0) for the offload cache.-# ----------------------------------------------------------------------------------------def _q4_0_expert_specs(config: ModelConfig) -> dict[str, tuple[tuple[int, ...], torch.dtype]]:- E = config.num_experts- H, I = config.hidden_size, config.moe_intermediate_size- return {- "gate_up": ((E, 2 * I, row_bytes(H, GGML_Q4_0)), torch.uint8),- "down": ((E, H, row_bytes(I, GGML_Q4_0)), torch.uint8),- }---def load_q4_0_expert_sources(- model_path: str, config: ModelConfig, *, layer_sink=None-) -> dict[str, list[torch.Tensor]]:- """Per-layer host banks of the routed experts' native Q4_0 block bytes.-- ``gate_up`` is one ``[E, 2I, H//32*18]`` tensor per layer and ``down`` one- ``[E, H, I//32*18]`` per layer (independent :class:`HostBank` allocations) -- each- expert's packed rows verbatim from the GGUF (no dequant), whole layers arriving in- one shot (gate_up + down = 2 writes/layer), so the offload cache streams whole- experts to the ggml MoE kernels.-- ``layer_sink=None`` (serving): pin each layer's two banks as they complete via an- internally-owned :class:`PinPipeline` (or, on a CUDA-less host, allocate the mmap- banks but never pin -- the CPU executor reads them pageable). ``layer_sink`` given- (converter): the completion tracker fires into it instead -- nothing here is pinned,- and the sink may release banks it has written out, so the returned tensors are only- valid until then (the caller owns that tradeoff).- """- from freetoken.models.gguf.reader import iter_gguf_tensors- from freetoken.moe.host_banks import LayerCompletionTracker, PinPipeline, alloc_layer_banks-- _require_tp1("expert banks")- L, E = config.num_layers, config.num_experts- H, I = config.hidden_size, config.moe_intermediate_size- h_bytes, i_bytes = row_bytes(H, GGML_Q4_0), row_bytes(I, GGML_Q4_0)- hb = alloc_layer_banks(_q4_0_expert_specs(config), L) # lazy anon mmaps (unpinned)- banks = {name: [b.tensor for b in hb[name]] for name in hb}- seen_gu, seen_dn = set(), set()-- def _load(sink) -> None:- tracker = LayerCompletionTracker(2, hb, sink) if sink is not None else None # gate_up + down- for t in iter_gguf_tensors(model_path):- if not t.name.startswith("blk."):- continue- layer = int(t.name.split(".")[1])- if t.name.endswith("ffn_gate_up_exps.weight"):- banks["gate_up"][layer].copy_(t.packed().reshape(E, 2 * I, h_bytes))- seen_gu.add(layer)- elif t.name.endswith("ffn_down_exps.weight"):- banks["down"][layer].copy_(t.packed().reshape(E, H, i_bytes))- seen_dn.add(layer)- else:- continue- if tracker is not None:- tracker.note(layer)-- if layer_sink is not None:- _load(layer_sink)- elif torch.cuda.is_available():- with PinPipeline() as pins:- _load(pins)- else:- _load(None) # CUDA-less: mmap banks stay pageable, never pinned-- want = set(range(L))- assert seen_gu == want and seen_dn == want, (- f"missing Q4_0 expert layers: gate_up {sorted(want - seen_gu)}, "- f"down {sorted(want - seen_dn)}"- )- return banks---def dummy_q4_0_expert_sources(config: ModelConfig) -> dict[str, list[torch.Tensor]]:- """Random Q4_0 expert banks shaped like ``load_q4_0_expert_sources`` output."""- from freetoken.moe.host_banks import alloc_layer_banks, pin_banks-- L = config.num_layers- hb = alloc_layer_banks(_q4_0_expert_specs(config), L)- banks = {name: [b.tensor for b in hb[name]] for name in hb}- for t in banks["gate_up"] + banks["down"]:- t.random_(0, 256)- if torch.cuda.is_available():- pin_banks(hb) # match the other dummies: pin-after-fill (no-op mmap fill on CPU-only)- return banks+ index = gguf_tensor_index(model_path)+ hidden = config.hidden_size++ def t(name: str):+ return index[name if name in index else f"{name}.weight"]++ embed = t("token_embd")+ yield from fused("model.embed_tokens", [embed], [matrix(embed)], hidden)+ if "output.weight" in index:+ out = t("output")+ yield from fused("lm_head", [out], [matrix(out)], hidden)+ yield "model.norm.weight", float_tensor(t("output_norm"))++ for layer in range(config.num_layers):+ blk, base = f"blk.{layer}", f"model.layers.{layer}"+ for suffix, rel in _LAYER_SCALAR_MAP.items():+ tensor = float_tensor(index[f"{blk}.{suffix}"])+ yield f"{base}.{rel}", tensor.reshape(1) if rel.endswith("layer_scalar") else tensor+ q, k = t(f"{blk}.attn_q"), t(f"{blk}.attn_k")+ qkv = [q, k, index.get(f"{blk}.attn_v.weight", k)]+ yield from fused(f"{base}.self_attn.qkv_proj", qkv, [matrix(p) for p in qkv], hidden)+ o = t(f"{blk}.attn_output")+ yield from fused(f"{base}.self_attn.o_proj", [o], [matrix(o)], int(o.ne[0]))+ gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))+ yield from fused(f"{base}.feed_forward.shared_mlp.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)+ yield from fused(f"{base}.feed_forward.shared_mlp.down_proj", [down], [matrix(down)], int(down.ne[0]))__all__ = ["parse_gguf_config","iter_gguf_weights",- "convert_gemma4_to_gguf",- "is_gguf_model",- "load_q4_0_expert_sources",- "dummy_q4_0_expert_sources",+ "to_gguf",]diff --git a/python/freetoken/models/gemma4/model.py b/python/freetoken/models/gemma4/model.pyindex 5c91b905..1b9d0da7 100644--- a/python/freetoken/models/gemma4/model.py+++ b/python/freetoken/models/gemma4/model.py@@ -61,6 +61,8 @@ class Gemma4Model(BaseOP):num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,embed_scale=config.embedding_scale,+ quant_config=config.quant,+ prefix=f"{prefix}.embed_tokens",)self.layers = OPList([@@ -91,13 +93,6 @@ class Gemma4ForCausalLM(BaseLLMModel):self._final_logit_softcapping = config.final_logit_softcappingsuper().__init__()- # GGUF checkpoints carry native block-quantized weights: swap the dense- # projections + embedding for GGUF-quant ops (experts stay on the offload cache).- from .gguf import convert_gemma4_to_gguf, is_gguf_model-- if is_gguf_model(config):- convert_gemma4_to_gguf(self, config)-def forward(self) -> torch.Tensor:output = self.model.forward(get_global_ctx().batch.input_ids)logits = self.lm_head.forward(output)diff --git a/python/freetoken/models/gemma4/weight.py b/python/freetoken/models/gemma4/weight.pyindex ba990c00..08293c7e 100644--- a/python/freetoken/models/gemma4/weight.py+++ b/python/freetoken/models/gemma4/weight.py@@ -196,7 +196,7 @@ def iter_weights(continue# Per-expert NVFP4 tensors go to the offload cache (expert pieces),- # not this dense pass; fused bf16/q4_0 experts lack ".experts.<int>." so are unaffected.+ # not this dense pass; fused bf16 experts lack ".experts.<int>." so are unaffected.if _NVFP4_EXPERT_RE.search(raw_name):continuediff --git a/python/freetoken/models/gguf/__init__.py b/python/freetoken/models/gguf/__init__.pyindex 75e40e4f..20a97b46 100644--- a/python/freetoken/models/gguf/__init__.py+++ b/python/freetoken/models/gguf/__init__.py@@ -1,26 +1,19 @@-from .dequant import GGML_NAME, dequantizefrom .reader import (FTW_METADATA_GGUF,- GgufTensor,gguf_architecture,gguf_config_source,gguf_tensor_names,is_gguf_path,- iter_gguf_tensors,load_gguf_metadata,write_metadata_gguf,)__all__ = [- "GGML_NAME",- "dequantize","FTW_METADATA_GGUF",- "GgufTensor","gguf_architecture","gguf_config_source","gguf_tensor_names","is_gguf_path",- "iter_gguf_tensors","load_gguf_metadata","write_metadata_gguf",]diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.pyindex 63b1a18b..4eb2a999 100644--- a/python/freetoken/models/gguf/config.py+++ b/python/freetoken/models/gguf/config.py@@ -18,6 +18,7 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names# reuses the model classes but a GGUF parse_config / iter_weights).GGUF_ARCH_TO_REGISTRY: dict[str, str] = {"gemma4": "Gemma4GGUFForCausalLM",+ "qwen4exp": "Qwen4ExpGGUFForCausalLM",}diff --git a/python/freetoken/models/gguf/dequant.py b/python/freetoken/models/gguf/dequant.pydeleted file mode 100644index 77c3ea01..00000000--- a/python/freetoken/models/gguf/dequant.py+++ /dev/null@@ -1,153 +0,0 @@-"""GGML block-quant dequantization in pure torch (the formats this repo's GGUF-checkpoints use: Q4_0, Q6_K, plus trivial F32/F16/BF16).--This is the *reference / CPU* path, NOT the engine's hot path: GGUF weights stay-packed and are dequantized inside the borrowed ggml CUDA kernels (see-``freetoken.kernel.gguf``). These routines are used only to (a) materialize the few-dense F32/F16 tensors at load (norms, scales, router) via :func:`dequantize`, and-(b) cross-check the CUDA kernels in tests. The ``BLOCK_SHAPE`` table and-:func:`row_bytes` are the type metadata the packed (kernel) path also relies on.--Each ``dequant_*`` takes the raw little-endian bytes as a ``uint8`` tensor whose-final axis spans whole blocks, and returns the values in *storage order* (ggml's-fastest axis first); the caller reshapes to the torch shape (``dims[::-1]``). The-math mirrors ``ggml-quants.c``.-"""--from __future__ import annotations--import torch--# ggml_type enum values (subset present in these checkpoints).-GGML_F32 = 0-GGML_F16 = 1-GGML_Q4_0 = 2-GGML_Q8_0 = 8-GGML_Q6_K = 14-GGML_BF16 = 30--# (block numel, bytes per block) per ggml type.-BLOCK_SHAPE: dict[int, tuple[int, int]] = {- GGML_F32: (1, 4),- GGML_F16: (1, 2),- GGML_BF16: (1, 2),- GGML_Q4_0: (32, 18),- GGML_Q8_0: (32, 34),- GGML_Q6_K: (256, 210),-}--GGML_NAME = {- GGML_F32: "F32",- GGML_F16: "F16",- GGML_BF16: "BF16",- GGML_Q4_0: "Q4_0",- GGML_Q8_0: "Q8_0",- GGML_Q6_K: "Q6_K",-}---def row_bytes(numel: int, ggml_type: int) -> int:- """Packed byte length of one row of ``numel`` elements in ``ggml_type`` blocks.-- Single source of truth for the ``numel // block * type_size`` math shared by the- packed-weight ops (``GGUFLinear``/``GGUFEmbedding``) and the expert bank loaders.- """- block, type_size = BLOCK_SHAPE[ggml_type]- assert numel % block == 0, (- f"{numel} not a multiple of block {block} for {GGML_NAME.get(ggml_type, ggml_type)}"- )- return numel // block * type_size---def _f16_scales(raw: torch.Tensor, lo: int, hi: int) -> torch.Tensor:- """Reinterpret bytes ``[lo:hi]`` (2 per block) of each block row as fp16 -> fp32 [N,1]."""- return raw[:, lo:hi].contiguous().view(torch.float16).to(torch.float32)---def dequant_q4_0(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor:- """Q4_0: per 32-elem block = fp16 scale ``d`` + 16 packed nibbles; ``w = d*(q-8)``.-- Byte ``j`` of the 16 holds element ``j`` in its low nibble and ``j+16`` in its high- nibble, so storage order within the block is ``[lo0..lo15, hi0..hi15]``.- """- raw = raw.reshape(-1, 18)- d = _f16_scales(raw, 0, 2) # [N,1]- qs = raw[:, 2:18] # [N,16] uint8- lo = (qs & 0x0F).to(torch.float32)- hi = (qs >> 4).to(torch.float32)- q = torch.cat([lo, hi], dim=1) # [N,32]- return ((q - 8.0) * d).reshape(-1).to(out_dtype)---def dequant_q6_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor:- """Q6_K: 256-elem super-block = 128B low nibbles + 64B high 2-bits + 16 int8- sub-scales + fp16 ``d``. Direct vectorization of ggml's two-half loop."""- raw = raw.reshape(-1, 210)- n = raw.shape[0]- ql = raw[:, 0:128] # [n,128]- qh = raw[:, 128:192] # [n,64]- sc = raw[:, 192:208].view(torch.int8).to(torch.float32) # [n,16]- d = _f16_scales(raw, 208, 210) # [n,1]-- y = torch.empty((n, 256), dtype=torch.float32, device=raw.device)- # l in 0..15 -> is=0; l in 16..31 -> is=1 (per ggml: is = l/16).- is_idx = (torch.arange(32, device=raw.device) // 16) # [32] in {0,1}- for h in range(2): # two 128-elem halves of the super-block- qlh = ql[:, h * 64:(h + 1) * 64] # [n,64]- qhh = qh[:, h * 32:(h + 1) * 32] # [n,32]- sch = sc[:, h * 8:(h + 1) * 8] # [n,8]- a = qlh[:, 0:32].to(torch.int32) # ql[l]- b = qlh[:, 32:64].to(torch.int32) # ql[l+32]- hb = qhh.to(torch.int32) # qh[l]- q1 = ((a & 0x0F) | (((hb >> 0) & 3) << 4)) - 32- q2 = ((b & 0x0F) | (((hb >> 2) & 3) << 4)) - 32- q3 = ((a >> 4) | (((hb >> 4) & 3) << 4)) - 32- q4 = ((b >> 4) | (((hb >> 6) & 3) << 4)) - 32- s1 = sch.index_select(1, is_idx + 0).to(torch.float32)- s2 = sch.index_select(1, is_idx + 2).to(torch.float32)- s3 = sch.index_select(1, is_idx + 4).to(torch.float32)- s4 = sch.index_select(1, is_idx + 6).to(torch.float32)- base = h * 128- y[:, base + 0:base + 32] = d * s1 * q1.to(torch.float32)- y[:, base + 32:base + 64] = d * s2 * q2.to(torch.float32)- y[:, base + 64:base + 96] = d * s3 * q3.to(torch.float32)- y[:, base + 96:base + 128] = d * s4 * q4.to(torch.float32)- return y.reshape(-1).to(out_dtype)---_DEQUANT = {- GGML_Q4_0: dequant_q4_0,- GGML_Q6_K: dequant_q6_k,-}---def dequantize(raw: torch.Tensor, ggml_type: int, out_dtype: torch.dtype) -> torch.Tensor:- """Dequantize ``raw`` (uint8) of any supported ggml type to flat ``out_dtype``."""- if ggml_type == GGML_F32:- return raw.view(torch.float32).to(out_dtype)- if ggml_type == GGML_F16:- return raw.view(torch.float16).to(out_dtype)- if ggml_type == GGML_BF16:- return raw.view(torch.bfloat16).to(out_dtype)- fn = _DEQUANT.get(ggml_type)- if fn is None:- raise NotImplementedError(- f"dequant for ggml type {GGML_NAME.get(ggml_type, ggml_type)} not implemented"- )- return fn(raw, out_dtype)---__all__ = [- "GGML_F32",- "GGML_F16",- "GGML_BF16",- "GGML_Q4_0",- "GGML_Q8_0",- "GGML_Q6_K",- "GGML_NAME",- "BLOCK_SHAPE",- "row_bytes",- "dequant_q4_0",- "dequant_q6_k",- "dequantize",-]diff --git a/python/freetoken/models/gguf/experts.py b/python/freetoken/models/gguf/experts.pynew file mode 100644index 00000000..0da56ceb--- /dev/null+++ b/python/freetoken/models/gguf/experts.py@@ -0,0 +1,104 @@+"""Routed-expert banks straight from a GGUF's ``blk.N.ffn_{gate,up,down}_exps`` tensors, in their packed ggml rows: pinned host banks for the offload cache, or device banks for resident experts.++Each layer keeps its own ggml types, so each layer gets its own bank shapes: ``gate_up`` [E, 2I, row_bytes] (expert e's gate rows, then its up rows) and ``down`` [E, H, row_bytes]; float experts keep their dtype ([E, 2I, H] and [E, H, I]).+A GGUF that stores ``ffn_gate_up_exps`` (gemma4) already holds the gate_up rows in that order."""++from __future__ import annotations++import contextlib+import os++import torch++from freetoken.layers.quantization import QuantKind+from freetoken.layers.quantization.ggml_types import float_dtype, mmq_tail_bytes, pad_rows, type_name+from freetoken.utils import init_logger++from .reader import GgufTensorInfo, gguf_tensor_index++logger = init_logger(__name__)+++def _expert_rows(info: GgufTensorInfo) -> torch.Tensor:+ """[E, rows, row_bytes] packed ggml rows, or [E, rows, K] for a float tensor."""+ k, rows, experts = info.ne+ raw = torch.from_numpy(info.data()).view(experts, rows, info.nbytes // (experts * rows))+ dtype = float_dtype(info.ggml_type)+ return raw if dtype is None else raw.view(dtype)+++def _drop_page_cache(info: GgufTensorInfo) -> None:+ try:+ fd = os.open(info.path, os.O_RDONLY)+ try:+ os.posix_fadvise(fd, info.offset, info.nbytes, os.POSIX_FADV_DONTNEED)+ finally:+ os.close(fd)+ except OSError:+ pass+++def _device_bank(shape: tuple[int, ...], k: int, dtype: torch.dtype, device: torch.device) -> torch.Tensor:+ """An [E, rows, row] device bank; a zero row follows the last one when ggml's MMQ tiles read past it."""+ experts, rows, row = shape+ flat = torch.empty(experts * rows + pad_rows(k), row, dtype=dtype, device=device)+ flat[experts * rows :].zero_()+ return flat[: experts * rows].view(shape)+++def load_gguf_expert_banks(model_path: str, num_layers: int, *, first_layer: int = 0, dummy: bool = False, device: torch.device | None = None):+ """``ExpertBanks`` with one bank per (layer, role): pinned host banks for the offload cache, filled from the file and pinned while later layers load, or resident banks on ``device``."""+ from freetoken.moe.expert_banks import ExpertBanks+ from freetoken.moe.host_banks import HostBank, PinPipeline++ index = gguf_tensor_index(model_path)+ sources: dict[str, list[torch.Tensor]] = {"gate_up": [], "down": []}+ tails: dict[str, list[int]] = {"gate_up": [], "down": []}+ tags = []+ with PinPipeline() if device is None else contextlib.nullcontext() as pins:+ for layer in range(num_layers):+ blk = f"blk.{first_layer + layer}"+ down = index[f"{blk}.ffn_down_exps.weight"]+ if f"{blk}.ffn_gate_up_exps.weight" in index:+ gate = index[f"{blk}.ffn_gate_up_exps.weight"]+ parts = [gate]+ else:+ gate, up = index[f"{blk}.ffn_gate_exps.weight"], index[f"{blk}.ffn_up_exps.weight"]+ if gate.ggml_type != up.ggml_type:+ raise NotImplementedError(f"{blk}: gate and up experts have different ggml types")+ parts = [gate, up]+ tags.append(("ggml", type_name(gate.ggml_type), type_name(down.ggml_type)))+ tails["gate_up"].append(mmq_tail_bytes(gate.ggml_type, gate.ne[0]))+ tails["down"].append(mmq_tail_bytes(down.ggml_type, down.ne[0]))+ rows = [_expert_rows(info) for info in parts]+ d = _expert_rows(down)+ gate_up_shape = (rows[0].shape[0], sum(r.shape[1] for r in rows), rows[0].shape[2])+ if device is None:+ host = {"gate_up": HostBank(gate_up_shape, rows[0].dtype), "down": HostBank(tuple(d.shape), d.dtype)}+ banks = {role: bank.tensor for role, bank in host.items()}+ else:+ banks = {+ "gate_up": _device_bank(gate_up_shape, gate.ne[0], rows[0].dtype, device),+ "down": _device_bank(tuple(d.shape), down.ne[0], d.dtype, device),+ }+ if dummy:+ banks["gate_up"].zero_()+ banks["down"].zero_()+ else:+ start = 0+ for r in rows:+ banks["gate_up"][:, start : start + r.shape[1]].copy_(r)+ start += r.shape[1]+ banks["down"].copy_(d)+ for info in (*parts, down):+ _drop_page_cache(info)+ if device is None:+ pins(layer, host)+ for role, bank in banks.items():+ sources[role].append(bank)+ total = sum(t.numel() * t.element_size() for ts in sources.values() for t in ts)+ logger.info_rank0(f"GGUF expert banks: {num_layers} layers, {total / 2**30:.1f} GiB {'on ' + str(device) if device is not None else 'in host RAM'}")+ return ExpertBanks("gguf", sources, kind=QuantKind.GGUF, kernel="ggml", layout=None, layer_tags=tags, row_tail_bytes=tails)+++__all__ = ["load_gguf_expert_banks"]diff --git a/python/freetoken/models/gguf/reader.py b/python/freetoken/models/gguf/reader.pyindex b950d929..2715f129 100644--- a/python/freetoken/models/gguf/reader.py+++ b/python/freetoken/models/gguf/reader.py@@ -2,22 +2,20 @@Thin layer over ``gguf.GGUFReader`` (gguf-py). Metadata is read into a plain dictkeyed by the GGUF field name (``general.architecture``, ``gemma4.block_count`` ...);-tensors are exposed as ``GgufTensor`` records carrying the *torch* shape (ggml dims-reversed), the ggml quant type, and a zero-copy ``uint8`` view of the packed block-bytes laid out as ``[rows, row_bytes]`` (rows = product of all but the fastest ggml-dim; row_bytes spans whole quant blocks of the fastest dim).+tensors are indexed as ``GgufTensorInfo`` records (ggml dims, ggml type, and where the+packed bytes sit in the memory-mapped file)."""from __future__ import annotationsimport functoolsimport os+import reimport structfrom dataclasses import dataclass-from typing import Any, Iterator+from typing import Anyimport numpy as np-import torchdef is_gguf_path(model_path: str) -> bool:@@ -65,7 +63,7 @@ def write_metadata_gguf(source_gguf: str, dest_path: str) -> None:Validates by re-parsing: the copy must list zero tensors and expose the identical KVkey set (the KV *bytes* are copied verbatim, so identical keys imply identical values)."""- import gguf+ from freetoken.third_party import ggufreader = gguf.GGUFReader(source_gguf)assert reader.tensors, f"{source_gguf}: no tensors to bound the KV section"@@ -98,20 +96,6 @@ def write_metadata_gguf(source_gguf: str, dest_path: str) -> None:)-@dataclass(frozen=True)-class GgufTensor:- name: str- shape: tuple[int, ...] # torch order (ggml dims reversed)- ggml_type: int- rows: int # product of shape[:-1] over the *ggml* layout = blocks-major rows- row_bytes: int # packed bytes per row (whole quant blocks of the fastest dim)- _raw: np.ndarray # uint8 view, shape [rows, row_bytes]-- def packed(self) -> torch.Tensor:- """Zero-copy ``[rows, row_bytes]`` uint8 tensor of the native block bytes."""- return torch.from_numpy(self._raw)--def _field_value(reader, name: str) -> Any:field = reader.fields.get(name)if field is None:@@ -121,7 +105,7 @@ def _field_value(reader, name: str) -> Any:@functools.cachedef _reader(model_path: str):- import gguf+ from freetoken.third_party import ggufreturn gguf.GGUFReader(model_path)@@ -133,6 +117,55 @@ def load_gguf_metadata(model_path: str) -> dict[str, Any]:return {name: field.contents() for name, field in reader.fields.items()}+_SPLIT_RE = re.compile(r"-(\d{5})-of-(\d{5})\.gguf$")+++def gguf_split_paths(model_path: str) -> tuple[str, ...]:+ """Every file of a split GGUF (``name-00001-of-00002.gguf``), in order; a single file is its own list."""+ count = int(_field_value(_reader(model_path), "split.count") or 1)+ if count == 1:+ return (model_path,)+ match = _SPLIT_RE.search(model_path)+ if match is None:+ raise ValueError(f"{model_path} is one of {count} GGUF splits but is not named like name-00001-of-{count:05d}.gguf")+ stem = model_path[: match.start()]+ paths = tuple(f"{stem}-{i:05d}-of-{count:05d}.gguf" for i in range(1, count + 1))+ missing = [p for p in paths if not os.path.isfile(p)]+ if missing:+ raise FileNotFoundError(f"GGUF split files missing: {missing}")+ return paths+++@dataclass(frozen=True)+class GgufTensorInfo:+ """Where one tensor's packed bytes sit: ``ne`` is in ggml order (fastest dim first)."""++ name: str+ path: str+ offset: int+ nbytes: int+ ggml_type: int+ ne: tuple[int, ...]++ def data(self) -> np.ndarray:+ """Zero-copy uint8 view of the bytes (memory-mapped)."""+ reader = _reader(self.path)+ return reader.data[self.offset : self.offset + self.nbytes]+++def gguf_tensor_index(model_path: str) -> dict[str, GgufTensorInfo]:+ """Every tensor of every split, by name."""+ index: dict[str, GgufTensorInfo] = {}+ for path in gguf_split_paths(model_path):+ for t in _reader(path).tensors:+ index[t.name] = GgufTensorInfo(+ t.name, path, int(t.data_offset), int(t.n_bytes), int(t.tensor_type), tuple(int(x) for x in t.shape)+ )+ return index++def gguf_architecture(model_path: str) -> str:arch = _field_value(_reader(model_path), "general.architecture")if arch is None:@@ -140,39 +173,8 @@ def gguf_architecture(model_path: str) -> str:return str(arch)-def iter_gguf_tensors(model_path: str) -> Iterator[GgufTensor]:- """Yield every tensor with its torch shape, ggml type, and packed block bytes."""- import gguf-- reader = _reader(model_path)- for t in reader.tensors:- ne = [int(s) for s in t.shape] # ggml order, fastest dim first- torch_shape = tuple(reversed(ne))- block, type_size = gguf.GGML_QUANT_SIZES[t.tensor_type]- n_fast = ne[0]- if n_fast % block != 0:- raise ValueError(- f"{t.name}: fastest dim {n_fast} not a multiple of block {block} "- f"for {t.tensor_type.name}"- )- row_bytes = n_fast // block * type_size- rows = int(np.prod(ne[1:])) if len(ne) > 1 else 1- # gguf-py returns quantized tensors as raw uint8 but F32/F16 as typed arrays;- # normalize everything to a flat byte view before shaping into [rows, row_bytes].- flat = np.ascontiguousarray(t.data).reshape(-1).view(np.uint8)- raw = flat.reshape(rows, row_bytes)- yield GgufTensor(- name=t.name,- shape=torch_shape,- ggml_type=int(t.tensor_type),- rows=rows,- row_bytes=row_bytes,- _raw=raw,- )--def gguf_tensor_names(model_path: str) -> set[str]:- return {t.name for t in _reader(model_path).tensors}+ return {t.name for path in gguf_split_paths(model_path) for t in _reader(path).tensors}__all__ = [@@ -181,9 +183,10 @@ __all__ = ["OUTPUT_WEIGHT_PRESENT_KV","gguf_config_source","write_metadata_gguf",- "GgufTensor",+ "GgufTensorInfo",+ "gguf_split_paths",+ "gguf_tensor_index","load_gguf_metadata","gguf_architecture",- "iter_gguf_tensors","gguf_tensor_names",]diff --git a/python/freetoken/models/gguf/tensors.py b/python/freetoken/models/gguf/tensors.pynew file mode 100644index 00000000..4cba1d08--- /dev/null+++ b/python/freetoken/models/gguf/tensors.py@@ -0,0 +1,56 @@+"""GGUF tensors as state-dict entries: float tensors in torch order, quantized ones as packed ggml rows grouped into the GGUF Linear's per-type segments."""++from __future__ import annotations++from typing import Iterator++import numpy as np+import torch++from freetoken.layers.quantization.ggml_types import is_quantized, pad_rows, segment_param, segments++from .reader import GgufTensorInfo++_FLOAT_DTYPES = {0: torch.float32, 1: torch.float16, 30: torch.bfloat16}+++def float_tensor(info: GgufTensorInfo) -> torch.Tensor:+ """A float tensor in torch order (ggml dims reversed)."""+ return torch.from_numpy(np.array(info.data())).view(_FLOAT_DTYPES[info.ggml_type]).view(*reversed(info.ne))+++def packed_rows(info: GgufTensorInfo) -> torch.Tensor:+ """Packed ggml rows ``[rows, row_bytes]`` of a quantized tensor."""+ rows = int(np.prod(info.ne[1:]))+ return torch.from_numpy(info.data()).view(rows, info.nbytes // rows)+++def matrix(info: GgufTensorInfo) -> torch.Tensor:+ return packed_rows(info) if is_quantized(info.ggml_type) else float_tensor(info)+++def fused(prefix: str, infos: list[GgufTensorInfo], parts: list[torch.Tensor], k: int, pad_to: int = 0) -> Iterator[tuple[str, torch.Tensor]]:+ """One fused module: float parts concatenated into ``weight``; with any quantized part, one matrix per ggml-type run (float runs as bf16 rows, quantized runs packed)."""+ types = [i.ggml_type for i in infos]+ pad = (-sum(p.shape[0] for p in parts)) % pad_to if pad_to else 0+ if not any(is_quantized(t) for t in types):+ rows = [p.to(torch.bfloat16) for p in parts]+ if pad:+ rows.append(torch.zeros(pad, *rows[0].shape[1:], dtype=torch.bfloat16))+ yield f"{prefix}.weight", torch.cat(rows, dim=0)+ return+ segs = segments(tuple(types), tuple(p.shape[0] for p in parts))+ for i, seg in enumerate(segs):+ rows = parts[seg.first_part : seg.first_part + seg.num_parts]+ zeros = pad if i == len(segs) - 1 else 0+ if is_quantized(seg.ggml_type):+ zeros += pad_rows(k)+ dtype = torch.uint8+ else:+ rows = [r.to(torch.bfloat16) for r in rows]+ dtype = torch.bfloat16+ tail = [torch.zeros(zeros, rows[0].shape[1], dtype=dtype)] if zeros else []+ yield f"{prefix}.{segment_param(i)}", torch.cat([*rows, *tail], dim=0)+++__all__ = ["float_tensor", "fused", "matrix", "packed_rows"]diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.pyindex 6d5481c1..e6a5d4ac 100644--- a/python/freetoken/models/gguf/tokenizer.py+++ b/python/freetoken/models/gguf/tokenizer.py@@ -13,7 +13,7 @@ from typing import Anyfrom .reader import gguf_architecture, load_gguf_metadata# GGUF architecture -> transformers GGUF tokenizer-converter key.-_TOKENIZER_ARCH = {"gemma4": "gemma4_text"}+_TOKENIZER_ARCH = {"gemma4": "gemma4_text", "qwen4exp": "qwen3"}def load_gguf_tokenizer(model_path: str):diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.pyindex 0fe4a4ff..15f866ca 100644--- a/python/freetoken/models/qwen4_exp/__init__.py+++ b/python/freetoken/models/qwen4_exp/__init__.py@@ -10,6 +10,7 @@ Contracts shared across modules (do not rename):"""from .config import parse_config+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguffrom .model import Qwen4ExpForCausalLM, Qwen4ExpForConditionalGenerationfrom .weight import (ftw_side_files,@@ -33,4 +34,7 @@ __all__ = ["load_ple_table","parse_config","iter_expert_pieces",+ "iter_gguf_weights",+ "parse_gguf_config",+ "to_gguf",]diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.pyindex e4137862..b1e5f9aa 100644--- a/python/freetoken/models/qwen4_exp/config.py+++ b/python/freetoken/models/qwen4_exp/config.py@@ -43,6 +43,8 @@ class Qwen4ExpArgs:index_budget: intindex_ratio: intimage_token_id: int | None = None+ # GGUF stores ssm_out with its value-head columns in llama.cpp's tiled order+ gdn_out_heads_tiled: bool = False@propertydef index_topk_blocks(self) -> int:diff --git a/python/freetoken/models/qwen4_exp/gdn.py b/python/freetoken/models/qwen4_exp/gdn.pyindex ed49dd42..48c92a8e 100644--- a/python/freetoken/models/qwen4_exp/gdn.py+++ b/python/freetoken/models/qwen4_exp/gdn.py@@ -32,9 +32,10 @@ class Qwen4ExpGatedDeltaNet(BaseOP):def __init__(self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,conv_kernel_size, rms_norm_eps, layer_id, output_gate: str = "sigmoid",- *, quant_config: QuantConfig | None = None, prefix: str = "",+ *, quant_config: QuantConfig | None = None, prefix: str = "", out_heads_tiled: bool = False,):self.layer_id = layer_id+ self._out_heads_tiled = out_heads_tiled# The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as# [V, K] while the LinearStatePool declares it [K, V]; these coincide (and the# hybrid-radix snapshot scatter h[h_row]->slot is a plain copy) only when the two head@@ -197,6 +198,10 @@ class Qwen4ExpGatedDeltaNet(BaseOP):core_out = core_out.reshape(-1, self.head_v_dim)z = z.reshape(-1, self.head_v_dim)out = self.norm.forward(core_out, z).reshape(total, -1)+ if self._out_heads_tiled:+ # grouped value heads (k-head major) -> tiled (k-head minor), the column order of a GGUF ssm_out+ r = self.num_v_heads // self.num_k_heads+ out = out.view(total, self.num_k_heads, r, self.head_v_dim).transpose(1, 2).reshape(total, -1)return self.out_proj.forward(out)diff --git a/python/freetoken/models/qwen4_exp/gguf.py b/python/freetoken/models/qwen4_exp/gguf.pynew file mode 100644index 00000000..7909324b--- /dev/null+++ b/python/freetoken/models/qwen4_exp/gguf.py@@ -0,0 +1,264 @@+"""Qwen3.8-Flash-Next from a llama.cpp ``qwen4exp`` GGUF: config from the GGUF metadata, tensor names, and the converter's transforms undone.++The llama.cpp converter (conversion/qwen.py, conversion/qwen4exp.py) stores every zero-centred norm as ``w + 1``, ``A_log`` as ``-exp(A_log)``, the squeezed conv kernels, the indexer projection split into q and k, and the GDN value heads in tiled order (k-head minor). Row reorders are undone on the packed bytes; ``ssm_out`` keeps its tiled columns and the GDN layer permutes its input instead (``Qwen4ExpArgs.gdn_out_heads_tiled``).+"""++from __future__ import annotations++import re+from dataclasses import replace+from types import SimpleNamespace+from typing import TYPE_CHECKING, Iterator++import torch++from freetoken.layers.quantization.ggml_types import is_quantized+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index+from freetoken.models.gguf.tensors import float_tensor, fused, matrix++from .config import parse_config++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+ from freetoken.models.gguf.config import GgufConfigShim++_ARCH = "qwen4exp"++# HF config fields the GGUF metadata does not carry (Qwen3.8-Flash-Next's own config.json values)+_FAMILY_DEFAULTS = dict(+ hidden_act="silu",+ output_gate_type="sigmoid",+ indexer_kv_heads=1,+ ngram_vocab_size_base=20000000,+ make_ngram_vocab_size_divisible_by=128,+ split_ngram_parts=128,+ norm_topk_prob=True,+)++_LAYER_LEAVES: dict[str, tuple[str, ...]] = {+ "linear_attn.in_proj_qkv": ("attn_qkv",),+ "linear_attn.in_proj_z": ("attn_gate",),+ "linear_attn.in_proj_b": ("ssm_beta",),+ "linear_attn.in_proj_a": ("ssm_alpha",),+ "linear_attn.out_proj": ("ssm_out",),+ "self_attn.q_proj": ("attn_q",),+ "self_attn.k_proj": ("attn_k",),+ "self_attn.v_proj": ("attn_v",),+ "self_attn.o_proj": ("attn_output",),+ "self_attn.indexer.index_qk_proj": ("indexer.q_proj", "indexer.k_proj"),+ "mlp.gate": ("ffn_gate_inp",),+ "mlp.shared_expert_gate": ("ffn_gate_inp_shexp",),+ "mlp.shared_expert.gate_proj": ("ffn_gate_shexp",),+ "mlp.shared_expert.up_proj": ("ffn_up_shexp",),+ "mlp.shared_expert.down_proj": ("ffn_down_shexp",),+ "mlp.experts.0.gate_proj": ("ffn_gate_exps",),+ "mlp.experts.0.up_proj": ("ffn_up_exps",),+ "mlp.experts.0.down_proj": ("ffn_down_exps",),+ "attn_hyper_connection.input_mix_weight_down": ("hc_attn_down",),+ "attn_hyper_connection.block_inject_weight": ("hc_attn_inject",),+ "attn_hyper_connection.input_mix_weight_up": ("hc_attn_up",),+ "mlp_hyper_connection.input_mix_weight_down": ("hc_ffn_down",),+ "mlp_hyper_connection.block_inject_weight": ("hc_ffn_inject",),+ "mlp_hyper_connection.input_mix_weight_up": ("hc_ffn_up",),+ "ple.key_proj": ("ple_key",),+ "ple.value_proj": ("ple_value",),+}+_TOP_LEAVES: dict[str, tuple[str, ...]] = {+ "model.language_model.embed_tokens": ("token_embd",),+ "lm_head": ("output",),+ "model.language_model.hyper_connection_mixer.input_mix_weight_down": ("output_hc_down",),+ "model.language_model.hyper_connection_mixer.input_mix_weight_up": ("output_hc_up",),+}+_LAYER_RE = re.compile(r"^model\.language_model\.layers\.(\d+)\.(.+)$")+++def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""+ if name in _TOP_LEAVES:+ return _TOP_LEAVES[name]+ m = _LAYER_RE.match(name)+ if m is None:+ return ()+ return tuple(f"blk.{m.group(1)}.{leaf}" for leaf in _LAYER_LEAVES.get(m.group(2), ()))+++def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:+ md = shim.metadata++ def kv(key: str):+ return md[f"{_ARCH}.{key}"]++ head_dim = int(kv("attention.key_length"))+ sections = [int(s) for s in kv("rope.dimension_sections")][:3]+ ratios = [int(r) for r in kv("attention.compress_ratios")]+ n_v = int(kv("ssm.time_step_rank"))+ text = SimpleNamespace(+ **_FAMILY_DEFAULTS,+ hidden_size=int(kv("embedding_length")),+ # block_count also counts the MTP layers a GGUF may carry after the decoder+ num_hidden_layers=int(kv("block_count")) - int(md.get(f"{_ARCH}.nextn_predict_layers", 0)),+ num_attention_heads=int(kv("attention.head_count")),+ num_key_value_heads=int(kv("attention.head_count_kv")),+ head_dim=head_dim,+ rope_parameters={+ "rope_type": "default",+ "rope_theta": float(kv("rope.freq_base")),+ "partial_rotary_factor": int(kv("rope.dimension_count")) / head_dim,+ "mrope_section": sections,+ "mrope_interleaved": True,+ },+ max_position_embeddings=int(kv("context_length")),+ full_attention_interval=int(kv("full_attention_interval")),+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),+ vocab_size=int(shim.vocab_size),+ tie_word_embeddings=bool(shim.tie_word_embeddings),+ num_experts=int(kv("expert_count")),+ num_experts_per_tok=int(kv("expert_used_count")),+ moe_intermediate_size=int(kv("expert_feed_forward_length")),+ shared_expert_intermediate_size=int(kv("expert_shared_feed_forward_length")),+ linear_num_key_heads=int(kv("ssm.group_count")),+ linear_num_value_heads=n_v,+ linear_key_head_dim=int(kv("ssm.state_size")),+ linear_value_head_dim=int(kv("ssm.inner_size")) // n_v,+ linear_conv_kernel_dim=int(kv("ssm.conv_kernel")),+ hc_count=int(kv("hyper_connection.count")),+ hc_lowrank=int(kv("hyper_connection.low_rank")),+ indexer_n_heads=int(kv("attention.indexer.head_count")),+ indexer_head_dim=int(kv("attention.indexer.key_length")),+ indexer_budget=int(kv("attention.indexer.top_k")),+ indexer_compress_ratio=max(ratios),+ ple_layer_ids=[int(i) + 1 for i in kv("ple.layers")],+ # the GGUF key is one n-gram head's row; HF counts all (ngram_size - 1) * heads_per_ngram heads+ ple_embed_dim=int(kv("embedding_length_per_layer_input")) * (int(kv("ple.ngram_size")) - 1) * int(kv("ple.heads_per_ngram")),+ ple_conv_kernel_size=int(kv("ple.conv_kernel")),+ ngram_size=int(kv("ple.ngram_size")),+ heads_per_ngram=int(kv("ple.heads_per_ngram")),+ eos_token_id=int(kv("ple.eos_token_id")),+ )+ image_token_id = md.get(f"{_ARCH}.ple.image_token_id")+ hf_like = SimpleNamespace(+ text_config=text,+ model_type="qwen4_exp",+ architectures=list(shim.architectures),+ image_token_id=None if image_token_id is None else int(image_token_id),+ vision_config=None,+ )+ config = parse_config(hf_like)+ return replace(config, expert_quant="gguf", qwen4_args=replace(config.qwen4_args, gdn_out_heads_tiled=True))+++# ----------------------------------------------------------------------------------------------+# Weights+# ----------------------------------------------------------------------------------------------++def _untile(t: torch.Tensor, num_k_heads: int, head_dim: int, dim: int = 0) -> torch.Tensor:+ """Tiled value heads (k-head minor) back to grouped (k-head major) along ``dim``."""+ n = t.shape[dim]+ r = n // (num_k_heads * head_dim)+ idx = torch.arange(n).view(r, num_k_heads, head_dim).transpose(0, 1).reshape(n)+ return t.index_select(dim, idx)+++def iter_gguf_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+ include_vision: bool = True,+) -> Iterator[tuple[str, torch.Tensor]]:+ """The dense state dict from the GGUF; the routed experts load as offload banks (``freetoken.models.gguf.experts``)."""+ if not include_non_moe:+ return+ from freetoken.utils import cached_load_hf_config++ config = parse_gguf_config(cached_load_hf_config(model_path))+ index = gguf_tensor_index(model_path)+ md = cached_load_hf_config(model_path).metadata+ g = config.linear_attention_group()+ hidden = config.hidden_size++ def t(name: str) -> GgufTensorInfo:+ return index[name if name in index else f"{name}.weight"]++ def norm(name: str) -> torch.Tensor:+ return float_tensor(t(name)).float() - 1.0++ def hc(prefix: str, src: str) -> Iterator[tuple[str, torch.Tensor]]:+ down, inject, up = (t(f"{src}_{p}") for p in ("down", "inject", "up"))+ yield from fused(f"{prefix}.input_mix_weight_down_block_inject", [down, inject], [matrix(down), matrix(inject)],+ hidden * config.qwen4_args.hc_count, pad_to=16)+ yield from fused(f"{prefix}.input_mix_weight_up", [up], [matrix(up)], config.qwen4_args.hc_lowrank)+ yield f"{prefix}.hc_norm.weight", norm(f"{src}_norm")++ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)+ for proj, k in (("down", hidden * config.qwen4_args.hc_count), ("up", config.qwen4_args.hc_lowrank)):+ info = t(f"output_hc_{proj}")+ yield from fused(f"model.hyper_connection_mixer.input_mix_weight_{proj}", [info], [matrix(info)], k)+ yield "model.hyper_connection_mixer.hc_norm.weight", norm("output_hc_norm")++ for layer in range(config.num_layers):+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"+ if config.is_linear_layer(layer):+ la = f"{pre}.linear_attn"+ nk, dk = g.num_key_heads, g.key_head_dim+ qkv, gate = t(f"{blk}.attn_qkv"), t(f"{blk}.attn_gate")+ qkv_rows = matrix(qkv)+ v_start = 2 * nk * dk+ qkv_rows = torch.cat([qkv_rows[:v_start], _untile(qkv_rows[v_start:], nk, g.value_head_dim)])+ z_rows = _untile(matrix(gate), nk, g.value_head_dim)+ beta_info, alpha_info = t(f"{blk}.ssm_beta"), t(f"{blk}.ssm_alpha")+ beta, alpha = (_untile(matrix(i), nk, 1) for i in (beta_info, alpha_info))+ if is_quantized(qkv.ggml_type) or is_quantized(gate.ggml_type):+ yield from fused(f"{la}.in_proj_qkvz", [qkv, gate], [qkv_rows, z_rows], hidden)+ yield from fused(f"{la}.in_proj_ba", [beta_info, alpha_info], [beta, alpha], hidden)+ else:+ yield from fused(f"{la}.in_proj", [qkv, gate, beta_info, alpha_info], [qkv_rows, z_rows, beta, alpha], hidden)+ conv = float_tensor(t(f"{blk}.ssm_conv1d")).float()+ conv = torch.cat([conv[:v_start], _untile(conv[v_start:], nk, g.value_head_dim)])+ yield f"{la}.conv1d.weight", conv.unsqueeze(1)+ yield f"{la}.dt_bias", _untile(float_tensor(t(f"{blk}.ssm_dt.bias")).float(), nk, 1)+ yield f"{la}.A_log", _untile(torch.log(-float_tensor(t(f"{blk}.ssm_a")).float()), nk, 1)+ yield f"{la}.norm.weight", float_tensor(t(f"{blk}.ssm_norm"))+ yield from fused(f"{la}.out_proj", [t(f"{blk}.ssm_out")], [matrix(t(f"{blk}.ssm_out"))], g.num_value_heads * g.value_head_dim)+ else:+ sa = f"{pre}.self_attn"+ parts = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]+ yield from fused(f"{sa}.qkv_proj", parts, [matrix(p) for p in parts], hidden)+ out = t(f"{blk}.attn_output")+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))+ yield f"{sa}.q_norm.weight", norm(f"{blk}.attn_q_norm")+ yield f"{sa}.k_norm.weight", norm(f"{blk}.attn_k_norm")+ iq, ik = t(f"{blk}.indexer.q_proj"), t(f"{blk}.indexer.k_proj")+ yield from fused(f"{sa}.indexer.index_qk_proj", [iq, ik], [matrix(iq), matrix(ik)], hidden)+ yield f"{sa}.indexer.q_layernorm.weight", norm(f"{blk}.indexer.q_norm")+ yield f"{sa}.indexer.k_layernorm.weight", norm(f"{blk}.indexer.k_norm")++ mlp = f"{pre}.mlp"+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))+ sg, su, sd = (t(f"{blk}.ffn_{p}_shexp") for p in ("gate", "up", "down"))+ yield from fused(f"{mlp}.shared_expert.gate_up_proj", [sg, su], [matrix(sg), matrix(su)], hidden)+ yield from fused(f"{mlp}.shared_expert.down_proj", [sd], [matrix(sd)], config.shared_expert_intermediate_size)+ yield f"{mlp}.shared_expert_gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp_shexp")).reshape(1, -1)+ yield from hc(f"{pre}.attn_hyper_connection", f"{blk}.hc_attn")+ yield from hc(f"{pre}.mlp_hyper_connection", f"{blk}.hc_ffn")++ if layer in config.qwen4_args.ple_layer_ids:+ ple = f"{pre}.ple"+ for proj in ("key", "value"):+ info = t(f"{blk}.ple_{proj}")+ yield from fused(f"{ple}.{proj}_proj", [info], [matrix(info)], config.qwen4_args.ple_embed_dim)+ for n in ("key", "query", "conv"):+ yield f"{ple}.norm_{n}.weight", norm(f"{blk}.ple_norm_{n}")+ yield f"{ple}.conv1d.weight", float_tensor(t(f"{blk}.ple_conv1d")).float().unsqueeze(1)+ yield f"{ple}.ple_embedding.layer_multipliers", torch.tensor([int(x) for x in md[f"{_ARCH}.ple.layer_multipliers"]], dtype=torch.int64)+ yield f"{ple}.ple_embedding.ngram_heads_vocab_sizes", torch.tensor([int(x) for x in md[f"{_ARCH}.ple.head_vocab_sizes"]], dtype=torch.int64)+ yield f"{ple}.ple_embedding.ngram_heads_offsets", torch.tensor([int(x) for x in md[f"{_ARCH}.ple.head_offsets"]], dtype=torch.int64)+++def ple_table_info(model_path: str) -> GgufTensorInfo:+ return gguf_tensor_index(model_path)["per_layer_token_embd.weight"]+++__all__ = ["iter_gguf_weights", "parse_gguf_config", "ple_table_info", "to_gguf"]diff --git a/python/freetoken/models/qwen4_exp/hc.py b/python/freetoken/models/qwen4_exp/hc.pyindex f13808dc..69aaa583 100644--- a/python/freetoken/models/qwen4_exp/hc.py+++ b/python/freetoken/models/qwen4_exp/hc.py@@ -98,6 +98,7 @@ class GatedResidual(BaseOP):self.pad_size = (-(self.lowrank + self.hc_count)) % 16self.input_mix_weight_down_block_inject = LinearReplicated(width, self.lowrank + self.hc_count + self.pad_size, has_bias=False,+ output_sizes=[self.lowrank, self.hc_count + self.pad_size],quant_config=config.quant, prefix=f"{prefix}.input_mix_weight_down_block_inject",)else:diff --git a/python/freetoken/models/qwen4_exp/model.py b/python/freetoken/models/qwen4_exp/model.pyindex 258badac..4eb69165 100644--- a/python/freetoken/models/qwen4_exp/model.py+++ b/python/freetoken/models/qwen4_exp/model.py@@ -52,6 +52,7 @@ def build_linear_mixer(config: ModelConfig, layer_id: int, prefix: str) -> BaseOoutput_gate=g.output_gate,quant_config=config.quant,prefix=prefix,+ out_heads_tiled=config.qwen4_args.gdn_out_heads_tiled,)@@ -92,6 +93,8 @@ class Qwen4ExpModel(BaseOP):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,+ quant_config=config.quant,+ prefix=f"{prefix}.embed_tokens",)self.layers = OPList([@@ -168,11 +171,8 @@ class Qwen4ExpForCausalLM(BaseLLMModel):return 0if engine_config.ple_backend == "disk":- from freetoken.utils import download_hf_weight-from .ple_disk import DiskRowTable, resolve_row_source- folder = download_hf_weight(engine_config.model_path)# one WAIT node per captured graph: the flag protocol supports a single consumeassert len(ple_layers) == 1, "disk PLE backend expects exactly one PLE layer"emb, args = ple_layers[0].ple_embedding, ple_layers[0].args@@ -186,7 +186,7 @@ class Qwen4ExpForCausalLM(BaseLLMModel):"image_token_id": args.image_token_id,}disk_table = DiskRowTable(- resolve_row_source(folder),+ resolve_row_source(engine_config.model_path),constants,max_graph_rows=max(256, engine_config.cuda_graph_max_bs or 0),max_extend_tokens=engine_config.max_extend_tokens,diff --git a/python/freetoken/models/qwen4_exp/ple_disk.py b/python/freetoken/models/qwen4_exp/ple_disk.pyindex 69ee9fa9..bb71da2f 100644--- a/python/freetoken/models/qwen4_exp/ple_disk.py+++ b/python/freetoken/models/qwen4_exp/ple_disk.py@@ -50,6 +50,9 @@ class PleRowSource:row_bytes: introw_stride: intscale: float+ # packed ggml rows (a GGUF table) of ``row_values`` values each; None for one fp8 byte per value+ ggml_type: int | None = None+ row_values: int | None = None@propertydef total_rows(self) -> int:@@ -95,9 +98,27 @@ def source_from_safetensors(folder: str) -> PleRowSource:return PleRowSource(paths, [f for f, _ in order], [b for _, b in order], rows, cols, cols, float(scale))-def resolve_row_source(folder: str) -> PleRowSource:+def source_from_gguf(model_path: str) -> PleRowSource:+ """The GGUF ``per_layer_token_embd`` table in place: one extent of packed ggml rows."""+ from freetoken.layers.quantization.ggml_types import row_bytes++ from .gguf import ple_table_info++ info = ple_table_info(model_path)+ values, rows = info.ne+ nbytes = row_bytes(info.ggml_type, values)+ return PleRowSource([info.path], [0], [info.offset], rows, nbytes, nbytes, 1.0, info.ggml_type, values)+++def resolve_row_source(model_path: str) -> PleRowSource:"""Pick the row source for a checkpoint; the seam where a repacked format would plug in."""- return source_from_safetensors(folder)+ from freetoken.models.gguf.reader import gguf_config_source++ if (gguf_src := gguf_config_source(model_path)) is not None:+ return source_from_gguf(gguf_src)+ from freetoken.utils import download_hf_weight++ return source_from_safetensors(download_hf_weight(model_path))class DiskRowTable:@@ -113,9 +134,14 @@ class DiskRowTable:dtype: torch.dtype = torch.bfloat16,) -> None:from freetoken.kernel.row_store import PleStore+ from freetoken.layers.quantization.ggml_types import float_dtypeself.num_rows = source.total_rows- self.head_dim = source.row_bytes # fp8: one byte per element+ self.head_dim = source.row_values or source.row_bytes # fp8: one byte per element+ self._ggml_type = source.ggml_type+ self._row_bytes = source.row_bytes+ # float rows (fp8, or F32 / F16 / BF16 in a GGUF) are only cast; quantized ggml rows go through ggml+ self._row_dtype = torch.float8_e4m3fn if source.ggml_type is None else float_dtype(source.ggml_type)self.dtype = dtypeself.heads = int(hash_constants["num_ngram_heads"])self.scale = source.scale@@ -142,7 +168,7 @@ class DiskRowTable:use_io_uring=os.getenv(_IO_URING_ENV, "1") != "0",)self._device = torch.device("cuda", torch.cuda.current_device())- self._token_bytes = self.heads * self.head_dim+ self._token_bytes = self.heads * self._row_bytes# allocated up front: pinned alloc inside stream capture is illegal; one replay consumes it at a timeself._graph_pinned = alloc_pinned_tensor(max_graph_rows * self._token_bytes, dtype=torch.uint8)self._graph_pinned.zero_() # padded decode lanes read whatever sits here@@ -272,7 +298,13 @@ class DiskRowTable:dev[:nbytes].copy_(pinned[:nbytes], non_blocking=True)if not capturing:self._eager_read[self._eager_slot].record(stream)- values = dev[:nbytes].view(torch.float8_e4m3fn).to(self.dtype)+ if self._row_dtype is not None:+ # copy=True: rows already in self.dtype must not alias the staging buffer the next fill reuses+ values = dev[:nbytes].view(self._row_dtype).to(self.dtype, copy=True)+ else:+ from freetoken.kernel import ggml++ values = ggml.dequantize(dev[:nbytes].view(-1, self._row_bytes), self._ggml_type, self.head_dim, self.dtype)if self.scale != 1.0:values = values * self.scalevalues = values.view(*row_ids.shape[:-1], -1)diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 678424ff..4b0cea25 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -213,6 +213,15 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {encoders=_QWEN_VL_ENCODERS,packed_modules_mapping=_QWEN4_EXP_PACKED,),+ # llama.cpp qwen4exp GGUF (text only): the same model classes with every tensor in its packed ggml type+ "Qwen4ExpGGUFForCausalLM": ModelSpec(+ "freetoken.models.qwen4_exp",+ "Qwen4ExpForCausalLM",+ parse_config="parse_gguf_config",+ iter_weights="iter_gguf_weights",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ packed_modules_mapping=_QWEN4_EXP_PACKED,+ ),# Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the# qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the# loader handles the compressed-tensors NVFP4 layout.@@ -282,12 +291,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {checkpoint_segments=_GEMMA4_SEGMENTS,packed_modules_mapping=_GEMMA4_PACKED,),- # GGUF (native Q4_0/Q6_K) gemma4: same model classes, GGUF config + weight loaders.+ # llama.cpp gemma4 GGUF (text only): the same model classes with every tensor in its packed ggml type"Gemma4GGUFForCausalLM": ModelSpec("freetoken.models.gemma4","Gemma4ForCausalLM",parse_config="parse_gguf_config",iter_weights="iter_gguf_weights",+ checkpoint_segments=_GEMMA4_SEGMENTS,+ packed_modules_mapping=_GEMMA4_PACKED,),"GptOssForCausalLM": ModelSpec("freetoken.models.gpt_oss",@@ -342,16 +353,19 @@ def _load_attr(module_path: str, attr_name: str) -> Any:def checkpoint_quant_config(model_path: str, hf_config: Any, spec: ModelSpec):- """The checkpoint's QuantConfig under the family's naming, or None for GGUF, whose native-quant ops the shared parser does not model yet."""- from freetoken.layers.quantization import NameMap, QuantConfig+ """The checkpoint's QuantConfig under the family's naming.+ A GGUF family gets the GGUF dialect over the file's tensor table, named through the ``to_gguf`` its module exports."""+ from freetoken.layers.quantization import GGUFConfig, NameMap, QuantConfig++ name_map = NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping)if spec.parse_config == "parse_gguf_config":- return None- return QuantConfig.from_hf(- hf_config,- name_map=NameMap(roots=spec.checkpoint_roots, segments=spec.checkpoint_segments, packed=spec.packed_modules_mapping),- unquantized=spec.unquantized_modules,- )+ to_gguf = _load_attr(spec.module, "to_gguf")+ from freetoken.models.gguf.reader import gguf_tensor_index++ types = {name: info.ggml_type for name, info in gguf_tensor_index(hf_config.model_path).items()}+ return GGUFConfig(types, to_gguf, name_map=name_map, unquantized=spec.unquantized_modules)+ return QuantConfig.from_hf(hf_config, name_map=name_map, unquantized=spec.unquantized_modules)def get_model_class(model_architecture: str, model_config: ModelConfig):diff --git a/python/freetoken/models/weight.py b/python/freetoken/models/weight.pyindex e3382b1b..b03266fb 100644--- a/python/freetoken/models/weight.py+++ b/python/freetoken/models/weight.py@@ -262,28 +262,9 @@ def ftw_lacks_vision(model_path: str) -> bool:return not any(name.startswith(VISION_KEY_PREFIXES) for name in ftw_tensor_names(model_path, "weight"))-def load_q4_0_moe_expert_sources(- model_path: str,- model_config,- *,- dummy: bool = False,- layer_sink=None,-) -> dict:- """Load (or fabricate, with ``dummy=True``) packed GGUF Q4_0 expert source banks.- ``layer_sink`` (converter) streams each completed layer's banks; ignored for dummy."""- _config, spec = _spec_for_model_path(model_path)- if dummy:- builder = _model_override(spec, "dummy_q4_0_expert_sources")- assert builder is not None, "model defines no dummy_q4_0_expert_sources"- return builder(model_config)- loader = _load_attr(spec.module, "load_q4_0_expert_sources")- return loader(model_path, model_config, layer_sink=layer_sink)--__all__ = ["load_weight","load_vision_weight","ftw_lacks_vision",- "load_q4_0_moe_expert_sources","iter_expert_tensors_parallel",]diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.pyindex 5521c34e..03ce4b75 100644--- a/python/freetoken/moe/expert_banks.py+++ b/python/freetoken/moe/expert_banks.py@@ -2,8 +2,8 @@The expert kernel (``QuantMethod.kernel``) owns the bank layout and the pack step; thecheckpoint side delivers pieces (``moe.expert_pieces``) and this module fills the pinned host-banks from them (``build_expert_banks``). The GGUF q4_0 experts still-use their own providers until they get a method.+banks from them (``build_expert_banks``). GGUF experts keep their packed ggml rows+(``models.gguf.experts``)."""from __future__ import annotations@@ -19,7 +19,7 @@ from freetoken.layers.quantization import QuantKindfrom freetoken.utils import init_loggerfrom .host_banks import alloc_layer_banks-from .offload_cache import _BANK_BYTES_PER_EXPERT, _BANK_SCHEMAS+from .offload_cache import _BANK_BYTES_PER_EXPERTlogger = init_logger(__name__)@@ -50,6 +50,8 @@ class ExpertBanks:layout: dict | None = None# per-layer quant tag (e.g. the gate_up and down GGUF types), for layers whose banks match in dtype and row shape but not in quant; None = one quantlayer_tags: list | None = None+ # per bank, per layer: bytes past an expert matrix's last row that the kernel reads and that must read as zeros (ggml MMQ); None = no such reads+ row_tail_bytes: dict[str, list[int]] | None = Nonedef _dummy_fill(role: str, tensor: torch.Tensor) -> None:@@ -161,46 +163,6 @@ def build_expert_banks(_PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader-def _q4_0_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks:- if parallel:- raise NotImplementedError(- "parallel reader not implemented for q4_0: GGUF is a single packed file "- "(not safetensors), so the common reader doesn't apply -- it needs a GGUF-native "- "parallel reader (parse the tensor table, chunked O_DIRECT over the one file)"- )- from freetoken.models.weight import load_q4_0_moe_expert_sources-- # Native GGUF Q4_0 routed experts: packed block bytes streamed to the GPU and- # dequantized inside the borrowed ggml MoE kernels (no bf16 expert copy). Banks are- # per-layer HostBanks (pin-after-fill), so conversion streams each completed layer's- # gate_up + down straight through the sink (dummy fabricates in one shot -> not streamed).- sink = None if dummy else layer_sink- sources = load_q4_0_moe_expert_sources(model_path, model_config, dummy=dummy, layer_sink=sink)- return ExpertBanks(- "q4_0", {name: sources[name] for name in _BANK_SCHEMAS["q4_0"]}, streamed=sink is not None- )---# expert formats that still load through their own provider (GGUF)-_PROVIDERS = {- "q4_0": _q4_0_banks,-}---def _legacy_expert_banks(model_path, model_config, device, dtype, dummy, parallel, workers, chunk, decode_target="gpu", layer_sink=None) -> ExpertBanks:- expert_quant = model_config.expert_quant- if expert_quant not in _PROVIDERS:- raise ValueError(- f"{expert_quant!r} experts load through their MoE quant method; "- f"only {sorted(_PROVIDERS)} still have a format provider"- )- return _PROVIDERS[expert_quant](- model_path, model_config, device, dtype, dummy,- parallel=parallel, workers=workers, chunk=chunk, decode_target=decode_target,- layer_sink=layer_sink,- )--def _method_expert_banks(model_path, model_config, method, device, dummy, parallel, workers, chunk, layer_sink=None, resident=False) -> ExpertBanks:from freetoken.moe.expert_pieces import iter_expert_pieces@@ -258,7 +220,7 @@ def bank_bytes_estimate(model_config, method=None) -> int | None:"""Estimated total expert-bank bytes of a raw checkpoint before loading it.With a bound expert ``method`` the kernel's layout gives the exact host bytes; otherwise the- format-tag table sizes the GGUF format. ``None`` for unknown formats or missing dims+ format-tag table sizes the format. ``None`` for unknown formats or missing dims(callers then skip the pre-load sizing)."""layers = getattr(model_config, "num_moe_layers", None)if method is not None and layers:@@ -287,12 +249,10 @@ def load_expert_banks(*,method=None,device: torch.device,- dtype: torch.dtype,dummy: bool = False,parallel: bool | None = None,workers: int = 8,chunk: int = _PARALLEL_CHUNK,- decode_target: str = "gpu",layer_sink=None,layer_residency: list[str] | None = None,resident: bool = False,@@ -315,8 +275,8 @@ def load_expert_banks(streamable formats, so callers must check ``streamed`` rather than assume it fired.``method`` (the bound expert quant method of the model's offload layers) selects- the generic path: the family's pieces packed by the method's kernel. Without it only the- GGUF q4_0 format loads, through its own provider.+ the generic path: the family's pieces packed by the method's kernel. Without it only an+ FTW checkpoint loads.``layer_residency``: per-layer ``HostResidency`` labels applied at settle time -- explicitly on the FTW fast path, ambiently (``requested_residency``) in the slow-path providers.Applied labels are echoed on ``ExpertBanks.layer_residency``; a loader that settles some other way leaves it ``None`` (CPU-layer decode still works on pinned banks, it just saves no pin quota).@@ -370,9 +330,14 @@ def load_expert_banks(from freetoken.moe.host_banks import requested_residencydef _build(par: bool) -> ExpertBanks:- if method is not None:- return _method_expert_banks(model_path, model_config, method, device, dummy, par, workers, chunk, layer_sink, resident)- return _legacy_expert_banks(model_path, model_config, device, dtype, dummy, par, workers, chunk, decode_target, layer_sink)+ if method is not None and method.kind is QuantKind.GGUF:+ from freetoken.models.gguf.experts import load_gguf_expert_banks++ # each layer keeps its own ggml types, so the banks come from the file, not from one method's layout+ return load_gguf_expert_banks(model_path, model_config.num_moe_layers, dummy=dummy, device=device if resident else None)+ if method is None:+ raise ValueError("the experts of a raw checkpoint are packed by the offload layers' MoE quant method; pass method=")+ return _method_expert_banks(model_path, model_config, method, device, dummy, par, workers, chunk, layer_sink, resident)with requested_residency(layer_residency) as residency_plan:try:diff --git a/python/freetoken/moe/fused_q4_0.py b/python/freetoken/moe/fused_q4_0.pydeleted file mode 100644index cdab82bf..00000000--- a/python/freetoken/moe/fused_q4_0.py+++ /dev/null@@ -1,53 +0,0 @@-"""Grouped expert GEMM over native GGUF Q4_0 banks (borrowed ggml MoE kernels).--Ports vLLM/sglang's ``_fused_moe_gguf`` MMVQ path onto FreeToken's offload-cache-interface: the experts are streamed to the GPU as packed Q4_0 block bytes and-dequantized *inside* ``ggml_moe_a8_vec`` -- no bf16 expert copy is materialized. We-use the MMVQ (vector) kernel for both prefill and decode: it consumes ``topk_ids``-directly (no ``moe_align_block_size`` needed) and on small batches it is the right-choice anyway. ``topk_ids`` already index the streamed cache slots (decode) or the-materialized layer positions (prefill).-"""--from __future__ import annotations--import torch--from freetoken.layers.activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul-from freetoken.models.gguf.dequant import GGML_Q4_0--_ACT = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}---def fused_experts_gguf_q4_0(- hidden_states: torch.Tensor,- gate_up_q: torch.Tensor, # [num_slots, 2I, H//32*18] uint8- down_q: torch.Tensor, # [num_slots, H, I//32*18] uint8- topk_weights: torch.Tensor,- topk_ids: torch.Tensor,- activation: str,-) -> torch.Tensor:- from freetoken.kernel.gguf import ggml_moe_a8_vec-- act_fn = _ACT.get(activation)- if act_fn is None:- raise ValueError(f"unsupported MoE activation {activation!r}")-- num_tokens = hidden_states.shape[0]- n2 = gate_up_q.shape[1] # 2 * intermediate- h = down_q.shape[1] # hidden- top_k = topk_ids.shape[1]- qt = int(GGML_Q4_0)-- # gate_up: [num_tokens*top_k, 2I] -> activation -> [num_tokens*top_k, I]- gate_up = ggml_moe_a8_vec(hidden_states, gate_up_q, topk_ids, top_k, qt, n2, num_tokens)- inter = act_fn(gate_up)- # down: each of the num_tokens*top_k intermediate rows uses its own expert id.- out = ggml_moe_a8_vec(inter, down_q, topk_ids, 1, qt, h, num_tokens * top_k)- out = out.reshape(num_tokens, top_k, h) * topk_weights.reshape(num_tokens, top_k, 1).to(- out.dtype- )- return out.sum(dim=1)---__all__ = ["fused_experts_gguf_q4_0"]diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex 701524e0..a209f9a9 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -46,9 +46,8 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {# Half the host/cache footprint of bf16; the grouped GEMM (kernel/triton/fp8_blockscale_moe)# reads the routed fp8 rows directly and dequantizes in the K-loop (no bf16 materialization)."fp8_block": ("gate_up", "gate_up_scale", "down", "down_scale"),- # native GGUF Q4_0 experts: packed block bytes per output row, dequantized inside- # the borrowed ggml MoE kernels. gate_up [L*E, 2I, H//32*18], down [L*E, H, I//32*18].- "q4_0": ("gate_up", "down"),+ # GGUF experts in their own ggml type per layer: gate_up [L*E, 2I, row_bytes] (gate rows, then up rows), down [L*E, H, row_bytes]; float experts keep their dtype+ "gguf": ("gate_up", "down"),# native ModelOpt rows for the Triton inline-dequant kernels: packed e2m1 codes +# fp8-e4m3 per-16 block scales + per-output-row fp16 globals (w1/w3 carry distinct# globals, and folding them into the e4m3 block scales would underflow)@@ -93,7 +92,6 @@ _BANK_BYTES_PER_EXPERT = {(2 * I // 128) * fp8_block_scale_pad(2 * I // 128, H // 128)+ (H // 128) * fp8_block_scale_pad(H // 128, I // 128)) * 2,- "q4_0": lambda H, I: 2 * I * (H // 32) * 18 + H * (I // 32) * 18,"nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2),"mxfp4": lambda H, I: 2 * I * (H // 2 + H // 32 + 2) + H * (I // 2 + I // 32 + 2),"ds_fp4": lambda H, I: 2 * I * (H // 2 + H // 32) + H * (I // 2 + I // 32),@@ -182,13 +180,12 @@ class OffloadMoeCache:# all layers = the plain --moe-strategy cpu case).self.cpu_layer_ids: frozenset = frozenset()# Bank row layouts, set by set_bank_sources: layouts[g][k] is (dtype, row shape) of bank k (layouts of two quant tags may be equal), layer_layout[l] is layer l's g.- # With one (dtype, row shape) per bank a bank cache is typed [cache_size, *row]; with several it is a byte segment each pool views with its own layout.+ # With one (dtype, row shape) per bank a bank cache is typed [cache_size, *row]; with several, or with pads, it is a byte segment each pool views with its own layout.self.layouts: list[tuple[tuple[torch.dtype, tuple[int, ...]], ...]] = []self.layer_layout: list[int] = [0] * self.num_layers+ # _pad_bytes[g][k]: zeroed bytes after the last row of every pool and staging buffer of layout g in bank k, for kernels that read past it (16-aligned)+ self._pad_bytes: list[list[int]] = []self._raw_banks = False- # Set by the engine when hybrid decode hands CPU routes to a kernel that reads slot 0 with weight 0 (no supports_inactive_slots).- # With several quants a staging buffer can leave another quant's bytes in a pool's slot 0, which may decode to NaN, so the last prefill layer zeroes those rows.- self.zero_inactive_route_slots = False# num_experts floor + nvfp4_marlin slot cap, shared with the runtime-rebuild path.self.validate_rebuild(self.cache_size, self.pool_sizes)assert not self.prefill_overlap or self.cache_size >= 2 * self.num_experts, (@@ -302,6 +299,8 @@ class OffloadMoeCache:# _stage_views[g][b] are buffer b's per-bank [num_experts, ...] views for layout g; _inval[b] are the global slots it overwrites.self._stage_bytes: list[int] = []self._stage_views: list[list[tuple[torch.Tensor, ...] | None]] = []+ # _stage_tails[g][b]: the pad after buffer b's rows in layout g+ self._stage_tails: list[list[list[torch.Tensor]]] = []self._inval: list[slice | torch.Tensor] = []self.prefill_copy_stream: torch.cuda.Stream | None = Noneself.prefill_begin_event: torch.cuda.Event | None = None@@ -337,6 +336,12 @@ class OffloadMoeCache:dtype, shape = self.layouts[layout][k]return cache[offset : offset + nrows * row].view(dtype).view(nrows, *shape)+ def _bank_bytes(self, k: int, pool_sizes: list[int]) -> int:+ """Bytes of raw bank ``k``: each pool's slots and pad, plus the largest pad, so two staging buffers fit wherever two layers of slots do."""+ layouts = [self._pool_layout(p) for p in range(len(pool_sizes))]+ slots = sum(s * self._row_bytes[g][k] + self._pad_bytes[g][k] for g, s in zip(layouts, pool_sizes))+ return slots + max(pads[k] for pads in self._pad_bytes)+def _stage_fits(self, buffers: int) -> bool:return all(c.numel() * c.element_size() >= buffers * s for (_, c), s in zip(self.banks, self._stage_bytes))@@ -356,14 +361,15 @@ class OffloadMoeCache:# rows[k][p]: bytes of one slot of pool p in bank k; before the banks exist, one unit per slot with a stage of Eif self.banks:rows = [[self._row_bytes[self._pool_layout(p)][k] for p in range(len(spans))] for k in range(len(self.banks))]+ pads = [[self._pad_bytes[self._pool_layout(p)][k] for p in range(len(spans))] for k in range(len(self.banks))]stage = self._stage_byteselse:- rows, stage = [[1] * len(spans)], [E]+ rows, pads, stage = [[1] * len(spans)], [[0] * len(spans)], [E]offsets = []- for row in rows:+ for row, pad in zip(rows, pads):offsets.append([0])- for s, r in zip(self.pool_sizes[:-1], row):- offsets[-1].append(offsets[-1][-1] + s * r)+ for s, r, d in zip(self.pool_sizes[:-1], row, pad):+ offsets[-1].append(offsets[-1][-1] + s * r + d)self._pool_offsets = offsets# local slots below this index overlap the prefill staging area in some bankself._pool_stable_from = [@@ -371,10 +377,8 @@ class OffloadMoeCache:for p, s in enumerate(self.pool_sizes)]self._inval = []- masks = []for b in range(2):mask = torch.zeros(self.cache_size, dtype=torch.bool)- masks.append(mask)for k in range(len(rows)):lo, hi = b * stage[k], (b + 1) * stage[k]for p, (base, s) in enumerate(spans):@@ -387,9 +391,8 @@ class OffloadMoeCache:else:self._inval.append(idx.to(self.device))self._stage_dirty = [False, False]- self._stage_slot0_rows = [[], []]if not self.banks:- self._pool_bank_views, self._stage_views = [], []+ self._pool_bank_views, self._stage_views, self._stage_tails = [], [], []returnself._pool_bank_views = [tuple(self._bank_rows(k, self._pool_layout(p), offsets[k][p], s) for k in range(len(self.banks)))@@ -402,11 +405,10 @@ class OffloadMoeCache:]for g in range(len(self.layouts))]- if len(self.layouts) > 1:- self._stage_slot0_rows = [- [self._bank_rows(k, self._pool_layout(p), offsets[k][p], 1) for p, base in enumerate(bases) if masks[b][base] for k in range(len(self.banks))]- for b in range(2)- ]+ self._stage_tails = [+ [[self.banks[k][1][b * stage[k] + E * row[k] :][: pad[k]] for k in range(len(self.banks)) if pad[k] and self._stage_fits(b + 1)] for b in range(2)]+ for row, pad in zip(self._row_bytes, self._pad_bytes)+ ]def stage_views(self, layer_id: int, buffer_id: int) -> tuple[torch.Tensor, ...]:"""Per-bank ``[num_experts, ...]`` views of prefill staging buffer ``buffer_id`` in ``layer_id``'s layout; row == expert id."""@@ -432,6 +434,7 @@ class OffloadMoeCache:sources: dict[str, list[torch.Tensor]],layer_residency: list[str] | None = None,layer_tags: list | None = None,+ row_tail_bytes: dict[str, list[int]] | None = None,) -> None:"""Attach the host (CPU pinned) expert source banks and allocate a GPU slotcache per bank, following the format's bank schema.@@ -447,6 +450,8 @@ class OffloadMoeCache:Non-pinned (LOCKED/PAGEABLE) layers have no device address: they must already be routed to the CPU executor (``cpu_layer_ids``, set BEFORE this call), the copy plan skips their rows, and their only movement is ``copy_missing``'s whole-layer pageable prefill branch -- which is why prefill overlap is incompatible with them.``layer_tags`` names each layer's quant (see :func:`bank_layouts`); layers of different tags never share a pool.++ ``row_tail_bytes`` gives, per bank and layer, how many bytes past the last row the kernel reads; every pool and staging buffer is then followed by that many zero bytes."""from freetoken.moe.legacy_format import canonical_rolefrom freetoken.moe.host_banks import HostResidency@@ -498,12 +503,22 @@ class OffloadMoeCache:for p in range(len(self.pool_sizes)):mixed = {self.layer_layout[l] for l, q in enumerate(self.pool_of_layer) if q == p}assert len(mixed) == 1, f"slot pool {p} holds layers of {len(mixed)} layouts; a pool needs one"- self._raw_banks = len(set(self.layouts)) > 1+ self._pad_bytes = [[0] * len(self.bank_schema) for _ in self.layouts]+ if row_tail_bytes is not None:+ tails = {canonical_role(name): per_layer for name, per_layer in row_tail_bytes.items()}+ for k, name in enumerate(self.bank_schema):+ for layer_id, tail in enumerate(tails[canonical_role(name)]):+ g = self.layer_layout[layer_id]+ self._pad_bytes[g][k] = max(self._pad_bytes[g][k], -(-tail // 16) * 16)+ self._raw_banks = len(set(self.layouts)) > 1 or any(map(any, self._pad_bytes))self._row_bytes = [[math.prod(shape) * dtype.itemsize for dtype, shape in layout] for layout in self.layouts]if self._raw_banks:# every pool view and staging buffer starts on a row boundary of its own layoutassert all(r % 16 == 0 for rows in self._row_bytes for r in rows), self._row_bytes- self._stage_bytes = [self.num_experts * max(rows[k] for rows in self._row_bytes) for k in range(len(self.bank_schema))]+ self._stage_bytes = [+ self.num_experts * max(rows[k] for rows in self._row_bytes) + max(pads[k] for pads in self._pad_bytes)+ for k in range(len(self.bank_schema))+ ]self._alloc_banks()if not self._stage_fits(1):raise ValueError(f"MoE cache of {self.cache_size} slots cannot stage one whole layer of experts")@@ -518,8 +533,7 @@ class OffloadMoeCache:def _alloc_banks(self) -> None:for k, name in enumerate(self.bank_schema):if self._raw_banks:- nbytes = sum(s * self._row_bytes[self._pool_layout(p)][k] for p, s in enumerate(self.pool_sizes))- self.bank_caches[name] = torch.zeros(nbytes, dtype=torch.uint8, device=self.device)+ self.bank_caches[name] = torch.zeros(self._bank_bytes(k, self.pool_sizes), dtype=torch.uint8, device=self.device)else:dtype, shape = self.layouts[0][k]self.bank_caches[name] = torch.zeros((self.cache_size, *shape), dtype=dtype, device=self.device)@@ -634,7 +648,7 @@ class OffloadMoeCache:)if self._raw_banks and pool_sizes is not None:for k, stage in enumerate(self._stage_bytes):- if sum(s * self._row_bytes[self._pool_layout(p)][k] for p, s in enumerate(pool_sizes)) < stage:+ if self._bank_bytes(k, pool_sizes) < stage:raise ValueError(f"moe_cache_size={cache_size} cannot stage one whole layer of experts")def rebuild(self, cache_size: int, pool_sizes: list[int] | None = None) -> None:@@ -663,7 +677,7 @@ class OffloadMoeCache:self.bank_caches = {}self._pool_bank_views = []self._stage_views = []- self._stage_slot0_rows = [[], []]+ self._stage_tails = []self.cache_size = cache_sizeself.pool_sizes = sizesif self.device.type == "cuda":@@ -796,7 +810,8 @@ class OffloadMoeCache:)self._prefill_hit_num = torch.zeros((1,), dtype=torch.int64, device=self.device)- def _invalidate_prefill_buffer(self, buffer_id: int) -> None:+ def _invalidate_prefill_buffer(self, buffer_id: int, layer_id: int) -> None:+ """Take buffer ``buffer_id``'s slots for staging ``layer_id``."""slots = self._inval[buffer_id]old_ids = self.id_of_slot[slots]self.slot_for_id.view(-1)[old_ids[old_ids >= 0].long()] = -1@@ -805,6 +820,9 @@ class OffloadMoeCache:# ensure_experts evicts them first.self.usage[slots] = 0self._stage_dirty[buffer_id] = True+ # the pad may hold rows of a wider layout or of a pool+ for tail in self._stage_tails[self.layer_layout[layer_id]][buffer_id]:+ tail.zero_()def begin_prefill(self) -> None:if not self.prefill_overlap:@@ -846,7 +864,7 @@ class OffloadMoeCache:)def copy() -> None:- self._invalidate_prefill_buffer(buffer_id)+ self._invalidate_prefill_buffer(buffer_id, layer_id)for (per_layer, _), view in zip(self.banks, self.stage_views(layer_id, buffer_id)):view.copy_(per_layer[layer_id], non_blocking=True)@@ -954,7 +972,7 @@ class OffloadMoeCache:with torch.cuda.stream(self.prefill_copy_stream):if self._prefill_buffer_has_release_event[buffer_id]:self.prefill_copy_stream.wait_event(self.prefill_release_events[buffer_id])- self._invalidate_prefill_buffer(buffer_id)+ self._invalidate_prefill_buffer(buffer_id, layer_id)if miss.size:run_starts = np.concatenate(([0], np.nonzero(np.diff(miss) != 1)[0] + 1))starts = miss[run_starts]@@ -997,12 +1015,13 @@ class OffloadMoeCache:return self.stage_views(layer_id, buffer_id)def release_prefill_layer(self, layer_id: int) -> None:- if self.zero_inactive_route_slots and layer_id == self.num_layers - 1:+ if layer_id == self.num_layers - 1:# the staged slots were invalidated and nothing registers them within the forward, so their bytes are free to clearfor b in range(2):if self._stage_dirty[b]:- for row in self._stage_slot0_rows[b]:- row.zero_()+ # staged rows left there may be shifted or of another quant, and kernels still read them: MMQ past a slot's last row, weight-0 routes on slot 0+ for (_, cache), stage in zip(self.banks, self._stage_bytes):+ cache.view(-1).view(torch.uint8)[b * stage : (b + 1) * stage].zero_()self._stage_dirty[b] = Falseif not self.prefill_overlap:returndiff --git a/python/freetoken/moe/offload_kernels.py b/python/freetoken/moe/offload_kernels.pyindex 1d863a2f..8c4ad8f3 100644--- a/python/freetoken/moe/offload_kernels.py+++ b/python/freetoken/moe/offload_kernels.py@@ -108,7 +108,7 @@ def materialize_layer(cache, layer_id: int) -> None:return# staging buffer 0 may overlap several pools, so it only holds the layer for this forward and registers nothingE = cache.num_experts- cache._invalidate_prefill_buffer(0)+ cache._invalidate_prefill_buffer(0, layer_id)rows = torch.arange(E, dtype=torch.int32, device=cache.device)cache.evict_slots[:E].copy_(rows)cache.src_indices[:E].copy_(rows)diff --git a/python/freetoken/third_party/__init__.py b/python/freetoken/third_party/__init__.pynew file mode 100644index 00000000..633cc8a7--- /dev/null+++ b/python/freetoken/third_party/__init__.py@@ -0,0 +1 @@+"""Vendored third-party Python code; each package carries its LICENSE and VERSION."""diff --git a/python/freetoken/third_party/gguf/LICENSE b/python/freetoken/third_party/gguf/LICENSEnew file mode 100644index 00000000..76f67efd--- /dev/null+++ b/python/freetoken/third_party/gguf/LICENSE@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2023 Georgi Gerganov++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/python/freetoken/third_party/gguf/VERSION b/python/freetoken/third_party/gguf/VERSIONnew file mode 100644index 00000000..f3ec79b9--- /dev/null+++ b/python/freetoken/third_party/gguf/VERSION@@ -0,0 +1,10 @@+llama.cpp 8c2134a8755d8a9792af513dab20282affe903e4 (2026-10-14), the same pin as kernel/csrc/ggml++Only the gguf-py modules that the GGUF reader and the ggml type tables need: gguf-py/gguf/{constants,gguf_reader,quants,lazy}.py.+The PyPI gguf release lags the ggml types in this tree (gguf 0.19.0 cannot read Q2_0 tensors), so FreeToken reads GGUF files with this copy.++Local changes, kept in freetoken.patch (already applied):+- __init__.py: imports only the vendored modules (upstream also pulls in the writer, vocab and metadata modules and their dependencies).+- gguf_reader.py: imports constants relatively, so the installed gguf package is never picked up.++To update: copy the same files from the llama.cpp commit that kernel/csrc/ggml pins, apply freetoken.patch, and update this file.diff --git a/python/freetoken/third_party/gguf/__init__.py b/python/freetoken/third_party/gguf/__init__.pynew file mode 100644index 00000000..f4c74ef9--- /dev/null+++ b/python/freetoken/third_party/gguf/__init__.py@@ -0,0 +1,4 @@+from .constants import *+from .lazy import *+from .gguf_reader import *+from .quants import *diff --git a/python/freetoken/third_party/gguf/constants.py b/python/freetoken/third_party/gguf/constants.pynew file mode 100644index 00000000..c31ab4a4--- /dev/null+++ b/python/freetoken/third_party/gguf/constants.py@@ -0,0 +1,6332 @@+from __future__ import annotations++from enum import Enum, IntEnum, auto+from typing import Any++#+# constants+#++GGUF_MAGIC = 0x46554747 # "GGUF"+GGUF_VERSION = 3+GGUF_DEFAULT_ALIGNMENT = 32+GGML_QUANT_VERSION = 2 # GGML_QNT_VERSION from ggml.h+GGML_MAX_DIMS = 4 # GGML_MAX_DIMS from ggml.h++#+# metadata keys+#+++class Keys:+ class General:+ TYPE = "general.type"+ ARCHITECTURE = "general.architecture"+ QUANTIZATION_VERSION = "general.quantization_version"+ ALIGNMENT = "general.alignment"+ FILE_TYPE = "general.file_type"++ # Per-tensor extra options (tensor name array + parallel option arrays, e.g. prec_a4).+ TENSOR_EXTRA_NAME = "general.tensor_extra.name"+ TENSOR_EXTRA_PREC_A4 = "general.tensor_extra.prec_a4"++ # Recommended Sampler Parameters+ SAMPLING_SEQUENCE = "general.sampling.sequence"+ SAMPLING_TOP_K = "general.sampling.top_k"+ SAMPLING_TOP_P = "general.sampling.top_p"+ SAMPLING_MIN_P = "general.sampling.min_p"+ SAMPLING_XTC_PROBABILITY = "general.sampling.xtc_probability"+ SAMPLING_XTC_THRESHOLD = "general.sampling.xtc_threshold"+ SAMPLING_TEMP = "general.sampling.temp"+ SAMPLING_PENALTY_LAST_N = "general.sampling.penalty_last_n"+ SAMPLING_PENALTY_REPEAT = "general.sampling.penalty_repeat"+ SAMPLING_MIROSTAT = "general.sampling.mirostat"+ SAMPLING_MIROSTAT_TAU = "general.sampling.mirostat_tau"+ SAMPLING_MIROSTAT_ETA = "general.sampling.mirostat_eta"++ # Authorship Metadata+ NAME = "general.name"+ AUTHOR = "general.author"+ VERSION = "general.version"+ ORGANIZATION = "general.organization"++ FINETUNE = "general.finetune"+ BASENAME = "general.basename"++ DESCRIPTION = "general.description"+ QUANTIZED_BY = "general.quantized_by"++ SIZE_LABEL = "general.size_label"++ # Licensing details+ LICENSE = "general.license"+ LICENSE_NAME = "general.license.name"+ LICENSE_LINK = "general.license.link"++ # Typically represents the converted GGUF repo (Unless native)+ URL = "general.url" # Model Website/Paper+ DOI = "general.doi"+ UUID = "general.uuid"+ REPO_URL = "general.repo_url" # Model Source Repository (git/svn/etc...)++ # Model Source during conversion+ SOURCE_URL = "general.source.url" # Model Website/Paper+ SOURCE_DOI = "general.source.doi"+ SOURCE_UUID = "general.source.uuid"+ SOURCE_REPO_URL = "general.source.repo_url" # Model Source Repository (git/svn/etc...)++ # Base Model Source. There can be more than one source if it's a merged+ # model like with 'Mistral-7B-Merge-14-v0.1'. This will assist in+ # tracing linage of models as it is finetuned or merged over time.+ BASE_MODEL_COUNT = "general.base_model.count"+ BASE_MODEL_NAME = "general.base_model.{id}.name"+ BASE_MODEL_AUTHOR = "general.base_model.{id}.author"+ BASE_MODEL_VERSION = "general.base_model.{id}.version"+ BASE_MODEL_ORGANIZATION = "general.base_model.{id}.organization"+ BASE_MODEL_DESCRIPTION = "general.base_model.{id}.description"+ BASE_MODEL_URL = "general.base_model.{id}.url" # Model Website/Paper+ BASE_MODEL_DOI = "general.base_model.{id}.doi"+ BASE_MODEL_UUID = "general.base_model.{id}.uuid"+ BASE_MODEL_REPO_URL = "general.base_model.{id}.repo_url" # Model Source Repository (git/svn/etc...)++ # Dataset Source+ DATASET_COUNT = "general.dataset.count"+ DATASET_NAME = "general.dataset.{id}.name"+ DATASET_AUTHOR = "general.dataset.{id}.author"+ DATASET_VERSION = "general.dataset.{id}.version"+ DATASET_ORGANIZATION = "general.dataset.{id}.organization"+ DATASET_DESCRIPTION = "general.dataset.{id}.description"+ DATASET_URL = "general.dataset.{id}.url" # Model Website/Paper+ DATASET_DOI = "general.dataset.{id}.doi"+ DATASET_UUID = "general.dataset.{id}.uuid"+ DATASET_REPO_URL = "general.dataset.{id}.repo_url" # Model Source Repository (git/svn/etc...)++ # Array based KV stores+ TAGS = "general.tags"+ LANGUAGES = "general.languages"++ class LLM:+ VOCAB_SIZE = "{arch}.vocab_size"+ CONTEXT_LENGTH = "{arch}.context_length"+ EMBEDDING_LENGTH = "{arch}.embedding_length"+ EMBEDDING_LENGTH_OUT = "{arch}.embedding_length_out"+ FEATURES_LENGTH = "{arch}.features_length"+ BLOCK_COUNT = "{arch}.block_count"+ LEADING_DENSE_BLOCK_COUNT = "{arch}.leading_dense_block_count"+ FEED_FORWARD_LENGTH = "{arch}.feed_forward_length"+ EXPERT_FEED_FORWARD_LENGTH = "{arch}.expert_feed_forward_length"+ EXPERT_SHARED_FEED_FORWARD_LENGTH = "{arch}.expert_shared_feed_forward_length"+ EXPERT_CHUNK_FEED_FORWARD_LENGTH = "{arch}.expert_chunk_feed_forward_length"+ USE_PARALLEL_RESIDUAL = "{arch}.use_parallel_residual"+ TENSOR_DATA_LAYOUT = "{arch}.tensor_data_layout"+ EXPERT_COUNT = "{arch}.expert_count"+ EXPERT_USED_COUNT = "{arch}.expert_used_count"+ EXPERT_SHARED_COUNT = "{arch}.expert_shared_count"+ EXPERT_GROUP_COUNT = "{arch}.expert_group_count"+ EXPERT_GROUP_USED_COUNT = "{arch}.expert_group_used_count"+ EXPERT_WEIGHTS_SCALE = "{arch}.expert_weights_scale"+ EXPERT_WEIGHTS_NORM = "{arch}.expert_weights_norm"+ EXPERT_GATING_FUNC = "{arch}.expert_gating_func"+ EXPERT_GROUP_SCALE = "{arch}.expert_group_scale"+ EXPERT_LATENT_LENGTH = "{arch}.expert_latent_length"+ EXPERTS_PER_GROUP = "{arch}.experts_per_group"+ MOE_EVERY_N_LAYERS = "{arch}.moe_every_n_layers"+ MOE_LATENT_SIZE = "{arch}.moe_latent_size"+ NEXTN_PREDICT_LAYERS = "{arch}.nextn_predict_layers"+ NUM_DEEPSTACK_LAYERS = "{arch}.n_deepstack_layers"+ DEEPSTACK_MAPPING = "{arch}.deepstack_mapping"+ POOLING_TYPE = "{arch}.pooling_type"+ LOGIT_SCALE = "{arch}.logit_scale"+ DECODER_START_TOKEN_ID = "{arch}.decoder_start_token_id"+ DECODER_BLOCK_COUNT = "{arch}.decoder_block_count"+ ATTN_LOGIT_SOFTCAPPING = "{arch}.attn_logit_softcapping"+ ROUTER_LOGIT_SOFTCAPPING = "{arch}.router_logit_softcapping"+ FINAL_LOGIT_SOFTCAPPING = "{arch}.final_logit_softcapping"+ SWIN_NORM = "{arch}.swin_norm"+ RESCALE_EVERY_N_LAYERS = "{arch}.rescale_every_n_layers"+ TIME_MIX_EXTRA_DIM = "{arch}.time_mix_extra_dim"+ TIME_DECAY_EXTRA_DIM = "{arch}.time_decay_extra_dim"+ RESIDUAL_SCALE = "{arch}.residual_scale"+ EMBEDDING_SCALE = "{arch}.embedding_scale"+ TOKEN_SHIFT_COUNT = "{arch}.token_shift_count"+ INTERLEAVE_MOE_LAYER_STEP = "{arch}.interleave_moe_layer_step"+ FULL_ATTENTION_INTERVAL = "{arch}.full_attention_interval"+ NUM_LOOPS = "{arch}.num_loops"+ SKIP_LOOP_FINAL_NORM = "{arch}.skip_loop_final_norm"+ HASH_LAYER_COUNT = "{arch}.hash_layer_count"+ ACTIVATION_SPARSITY_SCALE = "{arch}.activation_sparsity_scale"+ ALTUP_ACTIVE_IDX = "{arch}.altup.active_idx"+ ALTUP_NUM_INPUTS = "{arch}.altup.num_inputs"+ EMBD_LENGTH_PER_LAYER_INP = "{arch}.embedding_length_per_layer_input"+ SWIGLU_CLAMP_EXP = "{arch}.swiglu_clamp_exp"+ SWIGLU_CLAMP_SHEXP = "{arch}.swiglu_clamp_shexp"+ HIDDEN_ACT = "{arch}.hidden_activation"+ DENSE_FEAT_IN_SIZE = "{arch}.{dense}_feat_in"+ DENSE_FEAT_OUT_SIZE = "{arch}.{dense}_feat_out"+ TARGET_LAYERS = "{arch}.target_layers"+ TARGET_HIDDEN_SIZE = "{arch}.target_hidden_size"+ BLOCK_SIZE = "{arch}.block_size"+ CONV_KERNEL_SIZE = "{arch}.conv_kernel_size"+ CONV_GROUP_SIZE = "{arch}.conv_group_size"+ SELECTOR_RANK = "{arch}.selector_rank"+ SELECTOR_TOP_K = "{arch}.selector_top_k"+ SAMPLE_FROM_ANCHOR = "{arch}.sample_from_anchor"+ HAS_CONFIDENCE_HEAD = "{arch}.has_confidence_head"+ NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual"+ NORM_BEFORE_FC = "{arch}.norm_before_fc"++ class Adapters:+ COUNT = "{arch}.adapters.count"+ TOKEN_IDS_ACTIVATE = "{arch}.adapters.token_ids_activate"+ TOKEN_IDS_SUBSTITUTE = "{arch}.adapters.token_ids_substitute"+ LORA_RANK = "{arch}.adapters.lora_rank"+ ROUTER_GAIN = "{arch}.adapters.router_gain"++ class Attention:+ HEAD_COUNT = "{arch}.attention.head_count"+ HEAD_COUNT_KV = "{arch}.attention.head_count_kv"+ MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias"+ CLAMP_KQV = "{arch}.attention.clamp_kqv"+ KEY_LENGTH = "{arch}.attention.key_length"+ VALUE_LENGTH = "{arch}.attention.value_length"+ LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon"+ LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon"+ GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon"+ GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups"+ CAUSAL = "{arch}.attention.causal"+ Q_LORA_RANK = "{arch}.attention.q_lora_rank"+ KV_LORA_RANK = "{arch}.attention.kv_lora_rank"+ DECAY_LORA_RANK = "{arch}.attention.decay_lora_rank"+ ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank"+ VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank"+ GATE_LORA_RANK = "{arch}.attention.gate_lora_rank"+ REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count"+ SLIDING_WINDOW = "{arch}.attention.sliding_window"+ SCALE = "{arch}.attention.scale"+ OUTPUT_GROUP_COUNT = "{arch}.attention.output_group_count"+ OUTPUT_LORA_RANK = "{arch}.attention.output_lora_rank"+ OUTPUT_SCALE = "{arch}.attention.output_scale"+ VALUE_SCALE = "{arch}.attention.value_scale"+ COMPRESS_RATIOS = "{arch}.attention.compress_ratios"+ COMPRESS_ROPE_FREQ_BASE = "{arch}.attention.compress_rope_freq_base"+ TEMPERATURE_LENGTH = "{arch}.attention.temperature_length"+ KEY_LENGTH_MLA = "{arch}.attention.key_length_mla"+ VALUE_LENGTH_MLA = "{arch}.attention.value_length_mla"+ KEY_LENGTH_SWA = "{arch}.attention.key_length_swa"+ VALUE_LENGTH_SWA = "{arch}.attention.value_length_swa"+ KEY_LENGTH_MLA_SWA = "{arch}.attention.key_length_mla_swa"+ VALUE_LENGTH_MLA_SWA = "{arch}.attention.value_length_mla_swa"+ KV_LORA_RANK_SWA = "{arch}.attention.kv_lora_rank_swa"+ SHARED_KV_LAYERS = "{arch}.attention.shared_kv_layers"+ SLIDING_WINDOW_PATTERN = "{arch}.attention.sliding_window_pattern"+ RECURRENT_LAYERS = "{arch}.attention.recurrent_layers"+ TEMPERATURE_SCALE = "{arch}.attention.temperature_scale"+ ROPE_PATTERN = "{arch}.attention.rope_pattern"+ VALUE_EXPERT_COUNT = "{arch}.attention.value_expert_count"+ VALUE_EXPERT_USED_COUNT = "{arch}.attention.value_expert_used_count"++ class Indexer:+ HEAD_COUNT = "{arch}.attention.indexer.head_count"+ KEY_LENGTH = "{arch}.attention.indexer.key_length"+ TOP_K = "{arch}.attention.indexer.top_k"+ BLOCK_SIZE = "{arch}.attention.indexer.block_size" # MSA+ LOCAL_BLOCKS = "{arch}.attention.indexer.local_blocks" # MSA+ TYPES = "{arch}.attention.indexer.types"+ KPOOL = "{arch}.attention.indexer.kpool" # GLM5-Next+ KPOOL_SELECT_TAIL = "{arch}.attention.indexer.kpool_select_tail" # GLM5-Next++ class HyperConnection:+ COUNT = "{arch}.hyper_connection.count"+ SINKHORN_ITERATIONS = "{arch}.hyper_connection.sinkhorn_iterations"+ EPSILON = "{arch}.hyper_connection.epsilon"+ # scale of the post gate (DeepSeek-V4 hardcodes 2.0)+ MAGNITUDE = "{arch}.hyper_connection.magnitude"+ # absent means the mix projection is full rank (DeepSeek-V4 behaviour)+ LOW_RANK = "{arch}.hyper_connection.low_rank"++ class PerLayerEmbedding:+ LAYERS = "{arch}.ple.layers"+ NGRAM_SIZE = "{arch}.ple.ngram_size"+ HEADS_PER_NGRAM = "{arch}.ple.heads_per_ngram"+ CONV_KERNEL = "{arch}.ple.conv_kernel"+ LAYER_MULTIPLIERS = "{arch}.ple.layer_multipliers"+ HEAD_OFFSETS = "{arch}.ple.head_offsets"+ HEAD_VOCAB_SIZES = "{arch}.ple.head_vocab_sizes"+ EOS_TOKEN_ID = "{arch}.ple.eos_token_id"+ IMAGE_TOKEN_ID = "{arch}.ple.image_token_id"++ class Rope:+ DIMENSION_COUNT = "{arch}.rope.dimension_count"+ DIMENSION_COUNT_SWA = "{arch}.rope.dimension_count_swa"+ DIMENSION_SECTIONS = "{arch}.rope.dimension_sections"+ SECTION_ORDER = "{arch}.rope.section_order"+ FREQ_BASE = "{arch}.rope.freq_base"+ FREQ_BASE_SWA = "{arch}.rope.freq_base_swa"+ SCALING_TYPE = "{arch}.rope.scaling.type"+ SCALING_FACTOR = "{arch}.rope.scaling.factor"+ SCALING_ALPHA = "{arch}.rope.scaling.alpha"+ SCALING_ATTN_FACTOR = "{arch}.rope.scaling.attn_factor"+ SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length"+ SCALING_FINETUNED = "{arch}.rope.scaling.finetuned"+ SCALING_YARN_TRUNCATE = "{arch}.rope.scaling.yarn_truncate"+ SCALING_YARN_LOG_MUL = "{arch}.rope.scaling.yarn_log_multiplier"+ SCALING_YARN_EXT_FACTOR = "{arch}.rope.scaling.yarn_ext_factor"+ SCALING_YARN_ATTN_FACTOR = "{arch}.rope.scaling.yarn_attn_factor"+ SCALING_YARN_BETA_FAST = "{arch}.rope.scaling.yarn_beta_fast"+ SCALING_YARN_BETA_SLOW = "{arch}.rope.scaling.yarn_beta_slow"++ class Activation:+ SITU_BETA = "{arch}.activation.situ_beta"+ SITU_LINEAR_BETA = "{arch}.activation.situ_linear_beta"++ class AttnRes:+ BLOCK_SIZE = "{arch}.attn_res.block_size"++ class Split:+ LLM_KV_SPLIT_NO = "split.no"+ LLM_KV_SPLIT_COUNT = "split.count"+ LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"++ class HRM:+ LAYERS_PER_STACK = "{arch}.hrm.layers_per_stack"+ H_CYCLES = "{arch}.hrm.h_cycles"+ L_CYCLES = "{arch}.hrm.l_cycles"+ PREFIX_LM = "{arch}.hrm.prefix_lm"++ class SSM:+ CONV_KERNEL = "{arch}.ssm.conv_kernel"+ INNER_SIZE = "{arch}.ssm.inner_size"+ STATE_SIZE = "{arch}.ssm.state_size"+ TIME_STEP_RANK = "{arch}.ssm.time_step_rank"+ GROUP_COUNT = "{arch}.ssm.group_count"+ DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms"++ class KDA:+ HEAD_DIM = "{arch}.kda.head_dim"+ SAFE_GATE = "{arch}.kda.safe_gate"+ GATE_LOWER_BOUND = "{arch}.kda.gate_lower_bound"++ class WKV:+ HEAD_SIZE = "{arch}.wkv.head_size"++ class PosNet:+ EMBEDDING_LENGTH = "{arch}.posnet.embedding_length"+ BLOCK_COUNT = "{arch}.posnet.block_count"++ class ConvNext:+ EMBEDDING_LENGTH = "{arch}.convnext.embedding_length"+ BLOCK_COUNT = "{arch}.convnext.block_count"++ class Classifier:+ OUTPUT_LABELS = "{arch}.classifier.output_labels"+ POOLING_TYPE = "{arch}.classifier.pooling_type"+ ACTIVATION = "{arch}.classifier.activation"++ class ShortConv:+ L_CACHE = "{arch}.shortconv.l_cache"++ class Decision:+ TYPE = "{arch}.decision.type"+ # note: single-use-case keys can be hard-coded in cpp code+ BLOCK_COUNT = "{arch}.decision.block_count"+ ROUTING_BLOCK_COUNT = "{arch}.decision.routing_block_count"+ HEAD_COUNT = "{arch}.decision.head_count"+ MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens"+ TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "<type>" or "<type>.<n_opt bucket>"+ SCORE_BIAS = "{arch}.decision.score_bias.{n}.{i}" # added to the logit of level i of a score question with n levels++ class Tokenizer:+ MODEL = "tokenizer.ggml.model"+ PRE = "tokenizer.ggml.pre"+ LIST = "tokenizer.ggml.tokens"+ TOKEN_TYPE = "tokenizer.ggml.token_type"+ TOKEN_TYPE_COUNT = "tokenizer.ggml.token_type_count" # for BERT-style token types+ SCORES = "tokenizer.ggml.scores"+ MERGES = "tokenizer.ggml.merges"+ BOS_ID = "tokenizer.ggml.bos_token_id"+ EOS_ID = "tokenizer.ggml.eos_token_id"+ EOT_ID = "tokenizer.ggml.eot_token_id"+ EOM_ID = "tokenizer.ggml.eom_token_id"+ UNK_ID = "tokenizer.ggml.unknown_token_id"+ SEP_ID = "tokenizer.ggml.seperator_token_id"+ PAD_ID = "tokenizer.ggml.padding_token_id"+ MASK_ID = "tokenizer.ggml.mask_token_id"+ ADD_BOS = "tokenizer.ggml.add_bos_token"+ ADD_EOS = "tokenizer.ggml.add_eos_token"+ ADD_SEP = "tokenizer.ggml.add_sep_token"+ ADD_PREFIX = "tokenizer.ggml.add_space_prefix"+ REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces"+ PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap"+ SUPPRESS_TOKENS = "tokenizer.ggml.suppress_tokens"+ HF_JSON = "tokenizer.huggingface.json"+ RWKV = "tokenizer.rwkv.world"+ CHAT_TEMPLATE = "tokenizer.chat_template"+ CHAT_TEMPLATE_N = "tokenizer.chat_template.{name}"+ CHAT_TEMPLATES = "tokenizer.chat_templates"+ # Normalizer constants+ NORMALIZER_NFC = "tokenizer.ggml.normalizer.nfc"+ NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase"+ NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"+ # FIM/Infill special tokens constants+ FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id"+ FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id"+ FIM_MID_ID = "tokenizer.ggml.fim_mid_token_id"+ FIM_PAD_ID = "tokenizer.ggml.fim_pad_token_id"+ FIM_REP_ID = "tokenizer.ggml.fim_rep_token_id"+ FIM_SEP_ID = "tokenizer.ggml.fim_sep_token_id"+ # deprecated:+ PREFIX_ID = "tokenizer.ggml.prefix_token_id"+ SUFFIX_ID = "tokenizer.ggml.suffix_token_id"+ MIDDLE_ID = "tokenizer.ggml.middle_token_id"++ class Adapter:+ TYPE = "adapter.type"+ LORA_ALPHA = "adapter.lora.alpha"+ LORA_TASK_NAME = "adapter.lora.task_name"+ LORA_PROMPT_PREFIX = "adapter.lora.prompt_prefix"+ ALORA_INVOCATION_TOKENS = "adapter.alora.invocation_tokens"++ class IMatrix:+ CHUNK_COUNT = "imatrix.chunk_count"+ CHUNK_SIZE = "imatrix.chunk_size"+ DATASETS = "imatrix.datasets"++ class Clip:+ PROJECTOR_TYPE = "clip.projector_type"+ HAS_VISION_ENCODER = "clip.has_vision_encoder"+ HAS_AUDIO_ENCODER = "clip.has_audio_encoder"+ HAS_GEN_AUDIO_ENCODER = "clip.has_gen_audio_encoder"+ HAS_LLAVA_PROJECTOR = "clip.has_llava_projector"++ class ClipVision:+ PROJECTOR_TYPE = "clip.vision.projector_type" # for mixed modality models+ IMAGE_SIZE = "clip.vision.image_size"+ IMAGE_MIN_PIXELS = "clip.vision.image_min_pixels"+ IMAGE_MAX_PIXELS = "clip.vision.image_max_pixels"+ PREPROC_MIN_TILES = "clip.vision.preproc_min_tiles"+ PREPROC_MAX_TILES = "clip.vision.preproc_max_tiles"+ PREPROC_IMAGE_SIZE = "clip.vision.preproc_image_size"+ PATCH_SIZE = "clip.vision.patch_size"+ EMBEDDING_LENGTH = "clip.vision.embedding_length"+ FEED_FORWARD_LENGTH = "clip.vision.feed_forward_length"+ PROJECTION_DIM = "clip.vision.projection_dim"+ BLOCK_COUNT = "clip.vision.block_count"+ IMAGE_MEAN = "clip.vision.image_mean"+ IMAGE_STD = "clip.vision.image_std"+ MAX_SLICE_NUMS = "clip.vision.max_slice_nums"+ IMAGE_RESIZE_ALGO = "clip.vision.image_resize_algo"+ SPATIAL_MERGE_SIZE = "clip.vision.spatial_merge_size"+ SWIGLU_CLAMP = "clip.vision.swiglu_clamp"+ EXPERT_COUNT_PER_LAYER = "clip.vision.expert_count_per_layer" # dots3note pyramid MoE, 0 = dense layer+ EXPERT_USED_COUNT = "clip.vision.expert_used_count"+ USE_GELU = "clip.use_gelu"+ USE_SILU = "clip.use_silu"+ N_WA_PATTERN = "clip.vision.n_wa_pattern" # used by qwen2.5vl+ WA_LAYER_INDEXES = "clip.vision.wa_layer_indexes" # used by youtuvl+ WA_PATTERN_MODE = "clip.vision.wa_pattern_mode" # used by mimovl, per-layer -1/0/1+ IS_DEEPSTACK_LAYERS = "clip.vision.is_deepstack_layers"+ WINDOW_SIZE = "clip.vision.window_size"+ FEATURE_LAYERS = "clip.vision.feature_layer" # Granite4 Vision+ IMAGE_GRID_PINPOINTS = "clip.vision.image_grid_pinpoints" # Granite4 Vision++ class Attention:+ HEAD_COUNT = "clip.vision.attention.head_count"+ HEAD_COUNT_KV = "clip.vision.attention.head_count_kv" # used by mimovl (GQA)+ HEAD_DIM = "clip.vision.attention.head_dim" # set when qkv width != n_embd+ LAYERNORM_EPS = "clip.vision.attention.layer_norm_epsilon"++ class Projector:+ SCALE_FACTOR = "clip.vision.projector.scale_factor"+ QUERY_SIDE = "clip.vision.projector.query_side"+ WINDOW_SIDE = "clip.vision.projector.window_side"+ SPATIAL_OFFSETS = "clip.vision.projector.spatial_offsets"++ class SAM:+ BLOCK_COUNT = "clip.vision.sam.block_count"+ EMBEDDING_LENGTH = "clip.vision.sam.embedding_length"+ HEAD_COUNT = "clip.vision.sam.head_count"++ class ClipAudio:+ PROJECTOR_TYPE = "clip.audio.projector_type" # for mixed modality models+ NUM_MEL_BINS = "clip.audio.num_mel_bins"+ EMBEDDING_LENGTH = "clip.audio.embedding_length"+ FEED_FORWARD_LENGTH = "clip.audio.feed_forward_length"+ PROJECTION_DIM = "clip.audio.projection_dim"+ BLOCK_COUNT = "clip.audio.block_count"+ SUBSAMPLING_FACTOR = "clip.audio.subsampling_factor"+ CHUNK_SIZE = "clip.audio.chunk_size"+ CONV_KERNEL_SIZE = "clip.audio.conv_kernel_size"+ MAX_POS_EMB = "clip.audio.max_pos_emb"+ FEATURE_LAYERS = "clip.audio.feature_layer" # Granite Speech Plus+ RVQ_NUM_QUANTIZERS = "clip.audio.rvq.num_quantizers"+ RVQ_CODEBOOK_SIZE = "clip.audio.rvq.codebook_size"+ WA_PATTERN_MODE = "clip.audio.wa_pattern_mode" # per-layer -1 (full) / 0 (windowed)+ WINDOW_SIZE = "clip.audio.window_size"+ LOCAL_BLOCK_COUNT = "clip.audio.local_block_count" # mimo-v2.5: input_local_transformer layer count+ LOCAL_GROUP_SIZE = "clip.audio.local_group_size" # mimo-v2.5: input_local_transformer grouping size++ class Attention:+ HEAD_COUNT = "clip.audio.attention.head_count"+ LAYERNORM_EPS = "clip.audio.attention.layer_norm_epsilon"++ class Projector:+ STACK_FACTOR = "clip.audio.projector.stack_factor"+ WINDOW_SIZE = "clip.audio.projector.window_size"+ DOWNSAMPLE_RATE = "clip.audio.projector.downsample_rate"+ HEAD_COUNT = "clip.audio.projector.head_count"++ class ClipGenAudio:+ PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models+ # name of the weight variant, for settings that are not in the checkpoint+ MODEL_VARIANT = "clip.gen.audio.model_variant"+ EMBEDDING_LENGTH = "clip.gen.audio.embedding_length"+ FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length"+ BLOCK_COUNT = "clip.gen.audio.block_count"+ PROJECTION_DIM = "clip.gen.audio.projection_dim"++ class Attention:+ HEAD_COUNT = "clip.gen.audio.attention.head_count"+ HEAD_COUNT_KV = "clip.gen.audio.attention.head_count_kv"+ LAYERNORM_EPS = "clip.gen.audio.attention.layer_norm_epsilon"++ class Diffusion:+ SHIFT_LOGITS = "diffusion.shift_logits"++ class xIELU:+ ALPHA_P = "xielu.alpha_p"+ ALPHA_N = "xielu.alpha_n"+ BETA = "xielu.beta"+ EPS = "xielu.eps"++ class PrismHadamard:+ VERSION = "prism.hadamard.version"+ TIED_OUTPUT = "prism.hadamard.tied_output"+ BLOCK_SIZE = "prism.hadamard.block_size"+ TRANSFORM = "prism.hadamard.transform"+ AXIS = "prism.hadamard.axis"+ SIGN_MODE = "prism.hadamard.sign_mode"+ SIGN_WIDTHS = "prism.hadamard.sign_widths"+ SIGN_VALUES = "prism.hadamard.sign_values"+ WEIGHT_NAMES = "prism.hadamard.weight_names"+ INVERSE_WEIGHT_NAMES = "prism.hadamard.inverse_weight_names"+ GDN_V_GROUPED = "prism.hadamard.gdn_v_grouped"+++#+# recommended mapping of model tensor names for storage in gguf+#+++class GGUFType:+ MODEL = "model"+ ADAPTER = "adapter"+ IMATRIX = "imatrix"+ MMPROJ = "mmproj" # dummy, unused for now+++class MODEL_ARCH(IntEnum):+ MMPROJ = auto() # dummy arch for clip.cpp+ LLAMA = auto()+ LLAMA4 = auto()+ DECI = auto()+ FALCON = auto()+ FALCON_H1 = auto()+ BAICHUAN = auto()+ GROK = auto()+ GPT2 = auto()+ GPTJ = auto()+ GPTNEOX = auto()+ MPT = auto()+ STARCODER = auto()+ REFACT = auto()+ BERT = auto()+ MODERN_BERT = auto()+ NOMIC_BERT = auto()+ NOMIC_BERT_MOE = auto()+ NEO_BERT = auto()+ JINA_BERT_V2 = auto()+ JINA_BERT_V3 = auto()+ EUROBERT = auto()+ BLOOM = auto()+ STABLELM = auto()+ QWEN = auto()+ QWEN2 = auto()+ QWEN2MOE = auto()+ QWEN2VL = auto()+ QWEN3 = auto()+ QWEN3MOE = auto()+ QWEN3NEXT = auto()+ HRM_TEXT = auto()+ QWEN3VL = auto()+ QWEN3VLMOE = auto()+ QWEN35 = auto()+ QWEN35MOE = auto()+ CLEF = auto()+ VLLM_DECISION2 = auto()+ QWEN4EXP = auto()+ PHI2 = auto()+ PHI3 = auto()+ PHIMOE = auto()+ PLAMO = auto()+ PLAMO2 = auto()+ PLAMO3 = auto()+ CODESHELL = auto()+ ORION = auto()+ INTERNLM2 = auto()+ MINICPM = auto()+ MINICPM3 = auto()+ GEMMA = auto()+ GEMMA2 = auto()+ GEMMA3 = auto()+ GEMMA3N = auto()+ GEMMA4 = auto()+ GEMMA4_ASSISTANT = auto()+ GEMMA_EMBEDDING = auto()+ GEMMA_EMBEDDING2 = auto()+ STARCODER2 = auto()+ RWKV6 = auto()+ RWKV6QWEN2 = auto()+ RWKV7 = auto()+ ARWKV7 = auto()+ MAMBA = auto()+ MAMBA2 = auto()+ MAPLE = auto()+ JAMBA = auto()+ XVERSE = auto()+ COMMAND_R = auto()+ COHERE2 = auto()+ COHERE2MOE = auto()+ DBRX = auto()+ OLMO = auto()+ OLMO2 = auto()+ OLMOE = auto()+ MUSE_GLIMMER = auto()+ OPENELM = auto()+ ARCTIC = auto()+ DEEPSEEK = auto()+ DEEPSEEK2 = auto()+ DEEPSEEK2OCR = auto()+ DEEPSEEK32 = auto()+ DEEPSEEK4 = auto()+ CHATGLM = auto()+ GLM4 = auto()+ GLM4_MOE = auto()+ GLM_DSA = auto()+ GLM5_NEXT = auto()+ BITNET = auto()+ T5 = auto()+ T5ENCODER = auto()+ JAIS = auto()+ JAIS2 = auto()+ NEMOTRON = auto()+ NEMOTRON_H = auto()+ NEMOTRON_H_MOE = auto()+ EXAONE = auto()+ EXAONE4 = auto()+ EXAONE_MOE = auto()+ GRANITE = auto()+ GRANITE_MOE = auto()+ GRANITE_HYBRID = auto()+ GRANITE_SWITCH = auto()+ GRANITE_SWA = auto()+ CHAMELEON = auto()+ WAVTOKENIZER_DEC = auto()+ PLM = auto()+ BAILINGMOE = auto()+ BAILINGMOE2 = auto()+ BAILINGMOE3 = auto()+ DOTS1 = auto()+ DOTS3NOTE = auto()+ ARCEE = auto()+ AFMOE = auto()+ LAGUNA = auto()+ ERNIE4_5 = auto()+ ERNIE4_5_MOE = auto()+ HUNYUAN_MOE = auto()+ HUNYUAN_DENSE = auto()+ HUNYUAN_VL = auto()+ HY_V3 = auto()+ HY_V4 = auto()+ SMOLLM3 = auto()+ GPT_OSS = auto()+ LFM2 = auto()+ LFM2MOE = auto()+ DREAM = auto()+ SMALLTHINKER = auto()+ LLADA = auto()+ LLADA_MOE = auto()+ SEED_OSS = auto()+ GROVEMOE = auto()+ APERTUS = auto()+ COGVLM = auto()+ MINIMAX01 = auto()+ MINIMAXM2 = auto()+ MINIMAXM3 = auto()+ RND1 = auto()+ PANGU_EMBED = auto()+ MISTRAL3 = auto()+ EAGLE3 = auto()+ DFLASH = auto()+ MISTRAL4 = auto()+ PADDLEOCR = auto()+ MIMO2 = auto()+ STEP35 = auto()+ SPARK2_5 = auto()+ LLAMA_EMBED = auto()+ MAINCODER = auto()+ KIMI_LINEAR = auto()+ KIMI_K3 = auto()+ TALKIE = auto()+ MELLUM = auto()+ NANBEIGE = auto()+ QWEN3TTS = auto()+ POCKETTTS = auto()+ K2HORIZON = auto()+++class VISION_PROJECTOR_TYPE(IntEnum):+ MLP = auto()+ LDP = auto()+ LDPV2 = auto()+ RESAMPLER = auto()+ GLM_EDGE = auto()+ MERGER = auto()+ GEMMA3N = auto()+ GEMMA3 = auto()+ QWEN3VL = auto()+ LING3VL = auto()+ STEP3VL = auto()+ COGVLM = auto()+++class MODEL_TENSOR(IntEnum):+ TOKEN_EMBD = auto()+ TOKEN_EMBD_NORM = auto()+ MASKED_EMBD_CENTROIDS= auto()+ MASKED_EMBD_ORDERING = auto()+ TOKEN_TYPES = auto()+ POS_EMBD = auto()+ OUTPUT = auto()+ HRM_Z_L_INIT = auto()+ DENSE_2_OUT = auto() # embeddinggemma 2_Dense+ DENSE_3_OUT = auto() # embeddinggemma 3_Dense+ OUTPUT_NORM = auto()+ HC_HEAD_FN = auto()+ HC_HEAD_BASE = auto()+ HC_HEAD_SCALE = auto()+ HC_HEAD_NORM = auto() # qwen4exp+ HC_HEAD_DOWN = auto() # qwen4exp+ HC_HEAD_UP = auto() # qwen4exp+ ROPE_FREQS = auto()+ ROPE_FACTORS_LONG = auto()+ ROPE_FACTORS_SHORT = auto()+ ATTN_Q = auto()+ ATTN_K = auto()+ ATTN_V = auto()+ ATTN_QKV = auto()+ ATTN_OUT = auto()+ ATTN_NORM = auto()+ ATTN_NORM_2 = auto()+ ATTN_OUT_NORM = auto()+ ATTN_POST_NORM = auto()+ ATTN_ROT_EMBD = auto()+ ATTN_SINKS = auto()+ ATTN_GATE = auto()+ FFN_GATE_INP = auto()+ FFN_GATE_INP_SHEXP = auto()+ FFN_NORM = auto()+ FFN_PRE_NORM = auto() # alias of FFN_NORM+ FFN_PRE_NORM_2 = auto() # gemma4+ FFN_POST_NORM = auto()+ FFN_POST_NORM_1 = auto() # gemma4+ FFN_POST_NORM_2 = auto() # gemma4+ FFN_GATE = auto()+ FFN_DOWN = auto()+ FFN_UP = auto()+ FFN_ACT = auto()+ FFN_NORM_EXP = auto()+ FFN_GATE_EXP = auto()+ FFN_DOWN_EXP = auto()+ FFN_UP_EXP = auto()+ FFN_GATE_UP_EXP = auto()+ FFN_GATE_SHEXP = auto()+ FFN_DOWN_SHEXP = auto()+ FFN_UP_SHEXP = auto()+ FFN_GATE_CHEXP = auto()+ FFN_DOWN_CHEXP = auto()+ FFN_UP_CHEXP = auto()+ FFN_EXP_PROBS_B = auto()+ FFN_EXP_PROBS_B_VL = auto() # deepseek4 vision (bias for image tokens)+ FFN_GATE_TID2EID = auto()+ MOE_LATENT_DOWN = auto() # nemotron 3 super+ MOE_LATENT_UP = auto() # nemotron 3 super+ ATTN_Q_NORM = auto()+ ATTN_K_NORM = auto()+ LAYER_OUT_NORM = auto()+ LAYER_OUT_SCALE = auto()+ PER_LAYER_TOKEN_EMBD = auto() # gemma3n+ PER_LAYER_MODEL_PROJ = auto() # gemma3n+ PER_LAYER_INP_GATE = auto() # gemma3n+ PER_LAYER_PROJ = auto() # gemma3n+ PER_LAYER_PROJ_NORM = auto() # gemma3n+ PER_LAYER_POST_NORM = auto() # gemma3n+ ALTUP_PROJ = auto() # gemma3n+ ALTUP_UNEMBD_PROJ = auto() # gemma3n+ ALTUP_CORRECT_COEF = auto() # gemma3n+ ALTUP_CORRECT_SCALE = auto() # gemma3n+ ALTUP_PREDICT_COEF = auto() # gemma3n+ ALTUP_ROUTER = auto() # gemma3n+ ALTUP_ROUTER_NORM = auto() # gemma3n+ LAUREL_L = auto() # gemma3n+ LAUREL_R = auto() # gemma3n+ LAUREL_POST_NORM = auto() # gemma3n+ SSM_IN = auto()+ SSM_CONV1D = auto()+ SSM_X = auto()+ SSM_DT = auto()+ SSM_DT_NORM = auto()+ SSM_A = auto()+ SSM_B_NORM = auto()+ SSM_C_NORM = auto()+ SSM_D = auto()+ SSM_NORM = auto()+ SSM_OUT = auto()+ SSM_ALPHA = auto() # qwen3.5+ SSM_BETA_ALPHA = auto() # qwen3next+ SSM_CONV1D_Q = auto() # Kimi Linear+ SSM_CONV1D_K = auto() # Kimi Linear+ SSM_CONV1D_V = auto() # Kimi Linear+ SSM_F_A = auto() # Kimi Linear+ SSM_F_B = auto() # Kimi Linear+ SSM_BETA = auto() # Kimi Linear qwen3.5+ SSM_G_A = auto() # Kimi Linear+ SSM_G_B = auto() # Kimi Linear+ SSM_G = auto() # Kimi K3 (full-rank KDA gate, replaces SSM_G_A/SSM_G_B)+ ATTN_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, pre-attention)+ FFN_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, pre-FFN)+ OUTPUT_RES_SCORE = auto() # Kimi K3 (fused res_norm * res_proj, final)+ FFN_ROUTED_DOWN = auto() # Kimi K3 (latent MoE: hidden -> latent)+ FFN_ROUTED_UP = auto() # Kimi K3 (latent MoE: latent -> hidden)+ FFN_ROUTED_NORM = auto() # Kimi K3 (latent MoE: norm on expert output)+ TIME_MIX_W0 = auto()+ TIME_MIX_W1 = auto()+ TIME_MIX_W2 = auto()+ TIME_MIX_A0 = auto()+ TIME_MIX_A1 = auto()+ TIME_MIX_A2 = auto()+ TIME_MIX_V0 = auto()+ TIME_MIX_V1 = auto()+ TIME_MIX_V2 = auto()+ TIME_MIX_G1 = auto()+ TIME_MIX_G2 = auto()+ TIME_MIX_K_K = auto()+ TIME_MIX_K_A = auto()+ TIME_MIX_R_K = auto()+ TIME_MIX_LERP_X = auto()+ TIME_MIX_LERP_K = auto()+ TIME_MIX_LERP_V = auto()+ TIME_MIX_LERP_R = auto()+ TIME_MIX_LERP_G = auto()+ TIME_MIX_LERP_FUSED = auto()+ TIME_MIX_LERP_W = auto()+ TIME_MIX_FIRST = auto()+ TIME_MIX_DECAY = auto()+ TIME_MIX_DECAY_W1 = auto()+ TIME_MIX_DECAY_W2 = auto()+ TIME_MIX_KEY = auto()+ TIME_MIX_VALUE = auto()+ TIME_MIX_RECEPTANCE = auto()+ TIME_MIX_GATE = auto()+ TIME_MIX_LN = auto()+ TIME_MIX_OUTPUT = auto()+ CHANNEL_MIX_LERP_K = auto()+ CHANNEL_MIX_LERP_R = auto()+ CHANNEL_MIX_KEY = auto()+ CHANNEL_MIX_RECEPTANCE = auto()+ CHANNEL_MIX_VALUE = auto()+ ATTN_Q_A = auto()+ ATTN_Q_B = auto()+ ATTN_KV_A_MQA = auto()+ ATTN_KV_B = auto()+ ATTN_K_B = auto()+ ATTN_V_B = auto()+ ATTN_Q_A_NORM = auto()+ ATTN_KV_A_NORM = auto()+ ATTN_KV = auto()+ ATTN_KV_NORM = auto()+ ATTN_OUT_A = auto()+ ATTN_OUT_B = auto()+ HC_ATTN_FN = auto()+ HC_ATTN_BASE = auto()+ HC_ATTN_SCALE = auto()+ HC_FFN_FN = auto()+ HC_FFN_BASE = auto()+ HC_FFN_SCALE = auto()+ HC_ATTN_NORM = auto() # qwen4exp+ HC_ATTN_DOWN = auto() # qwen4exp+ HC_ATTN_UP = auto() # qwen4exp+ HC_ATTN_INJECT = auto() # qwen4exp+ HC_FFN_NORM = auto() # qwen4exp+ HC_FFN_DOWN = auto() # qwen4exp+ HC_FFN_UP = auto() # qwen4exp+ HC_FFN_INJECT = auto() # qwen4exp+ PLE_KEY = auto() # qwen4exp+ PLE_VALUE = auto() # qwen4exp+ PLE_NORM_KEY = auto() # qwen4exp+ PLE_NORM_QUERY = auto() # qwen4exp+ PLE_NORM_CONV = auto() # qwen4exp+ PLE_CONV1D = auto() # qwen4exp+ ATTN_COMPRESSOR_WKV = auto()+ ATTN_COMPRESSOR_WGATE = auto()+ ATTN_COMPRESSOR_APE = auto()+ ATTN_COMPRESSOR_NORM = auto()+ FFN_SUB_NORM = auto()+ ATTN_SUB_NORM = auto()+ DEC_ATTN_NORM = auto()+ DEC_ATTN_Q = auto()+ DEC_ATTN_K = auto()+ DEC_ATTN_V = auto()+ DEC_ATTN_OUT = auto()+ DEC_ATTN_REL_B = auto()+ DEC_CROSS_ATTN_NORM = auto()+ DEC_CROSS_ATTN_NORM_KV = auto()+ DEC_CROSS_ATTN_Q = auto()+ DEC_CROSS_ATTN_K = auto()+ DEC_CROSS_ATTN_V = auto()+ DEC_CROSS_ATTN_OUT = auto()+ DEC_CROSS_ATTN_REL_B = auto()+ DEC_FFN_NORM = auto()+ DEC_FFN_GATE = auto()+ DEC_FFN_DOWN = auto()+ DEC_FFN_UP = auto()+ DEC_OUTPUT_NORM = auto()+ ENC_ATTN_NORM = auto()+ ENC_ATTN_Q = auto()+ ENC_ATTN_K = auto()+ ENC_ATTN_V = auto()+ ENC_ATTN_OUT = auto()+ ENC_ATTN_REL_B = auto()+ ENC_FFN_NORM = auto()+ ENC_FFN_GATE = auto()+ ENC_FFN_DOWN = auto()+ ENC_FFN_UP = auto()+ ENC_OUTPUT_NORM = auto()+ CLS = auto() # classifier+ CLS_OUT = auto() # classifier output projection+ CLS_NORM = auto()+ DECISION_HIDDEN_NORM = auto()+ DECISION_PROJ_MEMORY = auto()+ DECISION_PROJ_QUESTION = auto()+ DECISION_PROJ_OPTION_QUESTION = auto()+ DECISION_PROJ_GLOBAL = auto()+ DECISION_PROJ_OPTION_CONTEXT = auto()+ DECISION_PROJ_OPTION_LEXICAL = auto()+ DECISION_OPTION_SUMMARY_NORM = auto()+ DECISION_FIELD_NORM = auto()+ DECISION_OPTION_NORM = auto()+ DECISION_SCALES = auto()+ DECISION_SCORER = auto()+ DECISION_SCORER_OUT = auto()+ DECISION_QUERY_NORM = auto()+ DECISION_KEY = auto()+ DECISION_QUERY = auto()+ DECISION_OPTION_MLP = auto()+ DECISION_QUERY_MLP = auto()+ CONV1D = auto()+ CONVNEXT_DW = auto()+ CONVNEXT_NORM = auto()+ CONVNEXT_PW1 = auto()+ CONVNEXT_PW2 = auto()+ CONVNEXT_GAMMA = auto()+ POSNET_CONV1 = auto()+ POSNET_CONV2 = auto()+ POSNET_NORM = auto()+ POSNET_NORM1 = auto()+ POSNET_NORM2 = auto()+ POSNET_ATTN_NORM = auto()+ POSNET_ATTN_Q = auto()+ POSNET_ATTN_K = auto()+ POSNET_ATTN_V = auto()+ POSNET_ATTN_OUT = auto()+ SHORTCONV_CONV = auto()+ SHORTCONV_INPROJ = auto()+ SHORTCONV_OUTPROJ = auto()+ VISEXP_ATTN_QKV = auto()+ VISEXP_ATTN_OUT = auto()+ VISEXP_GATE = auto()+ VISEXP_DOWN = auto()+ VISEXP_UP = auto()+ INDEXER_K_NORM = auto()+ INDEXER_PROJ = auto()+ INDEXER_ATTN_K = auto()+ INDEXER_ATTN_Q_B = auto()+ INDEXER_Q_PROJ = auto()+ INDEXER_K_PROJ = auto()+ INDEXER_Q_NORM = auto()+ INDEXER_COMPRESSOR_WKV = auto()+ INDEXER_COMPRESSOR_WGATE = auto()+ INDEXER_COMPRESSOR_APE = auto()+ INDEXER_COMPRESSOR_NORM = auto()+ INDEXER_KPOOL_GATE = auto()+ INDEXER_KPOOL_APE = auto()+ ATTN_V_GATE = auto() # k2-horizon MoVA router+ ATTN_V_EXP = auto() # k2-horizon MoVA value experts+ # vision+ V_MMPROJ = auto()+ V_MMPROJ_FC = auto()+ V_MMPROJ_MLP = auto()+ V_MMPROJ_PEG = auto()+ V_ENC_EMBD_CLS = auto()+ V_ENC_EMBD_PATCH = auto()+ V_ENC_EMBD_NORM = auto()+ V_ENC_EMBD_PATCH_NORM = auto() # allow multiple norms in the same embd, e.g. for gemma4u+ V_ENC_EMBD_POS = auto()+ V_ENC_INPUT_NORM = auto()+ V_ENC_ATTN_QKV = auto()+ V_ENC_ATTN_Q = auto()+ V_ENC_ATTN_Q_NORM = auto()+ V_ENC_ATTN_K = auto()+ V_ENC_ATTN_K_NORM = auto()+ V_ENC_ATTN_V = auto()+ V_ENC_ATTN_O = auto()+ V_ENC_ATTN_O_NORM = auto()+ V_ENC_ATTN_SINKS = auto() # mimovl+ V_ENC_POST_ATTN_NORM = auto()+ V_ENC_FFN_UP = auto()+ V_ENC_FFN_GATE = auto()+ V_ENC_FFN_DOWN = auto()+ V_ENC_FFN_GATE_INP = auto() # dots3note vision MoE router+ V_ENC_FFN_GATE_EXPS = auto()+ V_ENC_FFN_UP_EXPS = auto()+ V_ENC_FFN_DOWN_EXPS = auto()+ V_ENC_FFN_EXP_PROBS_B = auto()+ V_ENC_ATTN_POST_NORM = auto() # gemma4+ V_ENC_FFN_POST_NORM = auto()+ V_LAYER_SCALE_1 = auto()+ V_LAYER_SCALE_2 = auto()+ V_LAYER_OUT_SCALE = auto()+ V_PRE_NORM = auto()+ V_POST_NORM = auto()+ V_MM_PRE_NORM = auto() # hunyuanvl+ V_MM_POST_NORM = auto()+ V_MM_INP_NORM = auto()+ V_MM_INP_PROJ = auto() # gemma3+ V_MM_SOFT_EMB_NORM = auto() # gemma3+ V_MM_EMBEDDING = auto() # gemma3n+ V_MM_HARD_EMB_NORM = auto() # gemma3n+ V_ENC_CONV_STEM = auto() # gemma3n+ V_ENC_CONV_STEM_NORM = auto() # gemma3n+ V_ENC_MSFA_EXP = auto() # gemma3n+ V_ENC_MSFA_EXP_NORM = auto() # gemma3n+ V_ENC_MSFA_PROJ = auto() # gemma3n+ V_ENC_MSFA_PROJ_NORM = auto() # gemma3n+ V_ENC_MSFA_NORM = auto() # gemma3n+ V_RESMPL_POS_EMBD_K = auto() # minicpmv+ V_RESMPL_ATTN_Q = auto() # minicpmv+ V_RESMPL_ATTN_K = auto() # minicpmv+ V_RESMPL_ATTN_V = auto() # minicpmv+ V_RESMPL_ATTN_OUT = auto() # minicpmv+ V_RESMPL_KV = auto() # minicpmv+ V_RESMPL_KV_NORM = auto() # minicpmv+ V_RESMPL_POST_NORM = auto() # minicpmv+ V_RESMPL_Q_NORM = auto() # minicpmv+ V_RESMPL_PROJ = auto() # minicpmv+ V_RESMPL_QUERY = auto() # minicpmv+ V_TOK_EMBD_IMG_BREAK = auto() # pixtral+ V_TOK_EMBD_IMG_START = auto() # deepseek4v+ V_TOK_EMBD_IMG_END = auto() # deepseek4v+ V_TOK_EMBD_IMG_PAD = auto() # deepseek4v+ V_MM_PATCH_MERGER = auto() # mistral small 3.1+ V_DS_NORM = auto() # qwen3vl+ V_DS_FC1 = auto() # qwen3vl+ V_DS_FC2 = auto() # qwen3vl+ V_MERGER_LN1 = auto() # minicpmv4_6+ V_MERGER_ATTN_Q = auto() # minicpmv4_6+ V_MERGER_ATTN_K = auto() # minicpmv4_6+ V_MERGER_ATTN_V = auto() # minicpmv4_6+ V_MERGER_ATTN_O = auto() # minicpmv4_6+ V_MERGER_DS_LN = auto() # minicpmv4_6+ V_MERGER_DS_UP = auto() # minicpmv4_6+ V_MERGER_DS_DOWN = auto() # minicpmv4_6+ V_MM_POST_FC_NORM = auto() # cogvlm+ V_MM_UP = auto() # cogvlm+ V_MM_DOWN = auto() # cogvlm+ V_MM_GATE = auto() # cogvlm+ V_MM_MERGER_FC1 = auto() # minimax-m3 (patch-merge MLP)+ V_MM_MERGER_FC2 = auto() # minimax-m3 (patch-merge MLP)+ V_TOK_BOI = auto() # cogvlm+ V_TOK_EOI = auto() # cogvlm+ V_TOK_IMG_BEGIN = auto() # hunyuanvl+ V_TOK_IMG_END = auto() # hunyuanvl+ V_STD_BIAS = auto() # gemma4+ V_STD_SCALE = auto() # gemma4+ V_SAM_POS_EMBD = auto() # Deepseek-OCR+ V_SAM_PATCH_EMBD = auto() # Deepseek-OCR+ V_SAM_PRE_NORM = auto() # Deepseek-OCR+ V_SAM_POST_NORM = auto() # Deepseek-OCR+ V_SAM_ATTN_POS_H = auto() # Deepseek-OCR+ V_SAM_ATTN_POS_W = auto() # Deepseek-OCR+ V_SAM_ATTN_QKV = auto() # Deepseek-OCR+ V_SAM_ATTN_OUT = auto() # Deepseek-OCR+ V_SAM_MLP_LIN_1 = auto() # Deepseek-OCR+ V_SAM_MLP_LIN_2 = auto() # Deepseek-OCR+ V_SAM_NECK = auto() # Deepseek-OCR+ V_SAM_NET_2 = auto() # Deepseek-OCR+ V_SAM_NET_3 = auto() # Deepseek-OCR+ V_ENC_EMBD_IMGNL = auto() # Deepseek-OCR+ V_ENC_EMBD_VSEP = auto() # Deepseek-OCR+ V_TOK_EMBD_SEP = auto() # MiniCPM-V 4.7+ V_RESMPL_QUERY_768 = auto() # Deepseek-OCR-2+ V_RESMPL_QUERY_1024 = auto() # Deepseek-OCR-2++ # qformer projector (vision) - Granite4 Vision+ V_QF_PROJ_QUERY = auto()+ V_QF_PROJ_NORM = auto()+ V_QF_PROJ_LINEAR = auto()+ V_QF_SELF_ATTN_Q = auto()+ V_QF_SELF_ATTN_K = auto()+ V_QF_SELF_ATTN_V = auto()+ V_QF_SELF_ATTN_O = auto()+ V_QF_SELF_ATTN_NORM = auto()+ V_QF_CROSS_ATTN_Q = auto()+ V_QF_CROSS_ATTN_K = auto()+ V_QF_CROSS_ATTN_V = auto()+ V_QF_CROSS_ATTN_O = auto()+ V_QF_CROSS_ATTN_NORM = auto()+ V_QF_FFN_UP = auto()+ V_QF_FFN_DOWN = auto()+ V_QF_FFN_NORM = auto()+ V_PROJ_NORM = auto()+ # multi-projector (bid => projector id) - Granite4 vision+ V_MULTI_PROJ_IMG_POS = auto()+ V_MULTI_PROJ_QUERY = auto()+ V_MULTI_PROJ_NORM = auto()+ V_MULTI_PROJ_LINEAR = auto()+ V_MULTI_PROJ_POST_NORM = auto()++ # audio (mtmd)+ A_ENC_EMBD_POS = auto()+ A_ENC_EMBD_NORM = auto()+ A_ENC_EMBD_TO_LOGITS = auto() # lfm2+ A_ENC_INP_PROJ = auto() # gemma4+ A_ENC_CONV1D = auto()+ A_ENC_CONV1D_NORM = auto() # gemma3n+ A_ENC_CONV2D = auto()+ A_ENC_CONV_OUT = auto()+ A_PRE_NORM = auto()+ A_POST_NORM = auto()+ A_ENC_LAYER_PRE_NORM = auto() # gemma3n+ A_ENC_ATTN_Q = auto()+ A_ENC_ATTN_K = auto()+ A_ENC_ATTN_V = auto()+ A_ENC_ATTN_POST_NORM = auto()+ A_ENC_ATTN_PRE_NORM = auto()+ A_ENC_ATTN_K_REL = auto() # gemma4+ A_ENC_PER_DIM_SCALE = auto() # gemma3n+ A_ENC_INPUT_NORM = auto()+ A_ENC_OUTPUT = auto() # TODO @ngxson: rename to ATTN_OUT+ A_ENC_OUTPUT_NORM = auto() # TODO @ngxson: rename to ATTN_OUT+ A_ENC_FFN_UP = auto()+ A_ENC_FFN_NORM = auto()+ A_ENC_FFN_POST_NORM = auto() # gemma3n+ A_ENC_FFN_SCALE = auto() # gemma3n+ A_ENC_FFN_GATE = auto()+ A_ENC_FFN_DOWN = auto()+ A_ENC_FFN_UP_1 = auto() # lfm2, gemma3n+ A_ENC_FFN_NORM_1 = auto() # lfm2, gemma3n (pre-norm)+ A_ENC_FFN_POST_NORM_1 = auto() # gemma3n+ A_ENC_FFN_SCALE_1 = auto() # gemma3n+ A_ENC_FFN_GATE_1 = auto() # lfm2, gemma3n+ A_ENC_FFN_DOWN_1 = auto() # lfm2, gemma3n+ A_ENC_DOWNSAMPLE_CONV = auto() # mimo-audio-tokenizer: post-transformer downsample conv+ A_ENC_DOWNSAMPLE_NORM = auto() # mimo-audio-tokenizer: post-transformer downsample norm+ A_ENC_RVQ_CODEBOOK = auto() # mimo-audio-tokenizer: residual vector quantizer codebook, per quantizer index+ A_ENC_CONV_RES2 = auto() # qwen3tts+ A_ENC_SE_CONV1 = auto() # qwen3tts+ A_ENC_SE_CONV2 = auto() # qwen3tts+ A_ENC_ASP_ATTN = auto() # qwen3tts+ A_ENC_ASP_TDNN = auto() # qwen3tts+ # qwen3tts code_predictor: predicts the remaining RVQ codebooks+ A_GEN_CODE_PROJ_IN = auto() # small_to_mtp_projection+ A_GEN_CODE_EMBD = auto() # per-codebook embedding table, merged 3D [n_codebooks, vocab, dim]+ A_GEN_CODE_HEAD = auto() # per-codebook output head, merged 3D [n_codebooks, vocab, dim]+ A_GEN_CODE_OUT_EMBD = auto() # codebook-0 embedding, re-fed into the talker backbone (talker.model.codec_embedding)+ A_GEN_CODE_ATTN_NORM = auto()+ A_GEN_CODE_ATTN_Q = auto()+ A_GEN_CODE_ATTN_Q_NORM = auto()+ A_GEN_CODE_ATTN_K = auto()+ A_GEN_CODE_ATTN_K_NORM = auto()+ A_GEN_CODE_ATTN_V = auto()+ A_GEN_CODE_ATTN_OUT = auto()+ A_GEN_CODE_FFN_NORM = auto()+ A_GEN_CODE_FFN_GATE = auto()+ A_GEN_CODE_FFN_UP = auto()+ A_GEN_CODE_FFN_DOWN = auto()+ A_GEN_CODE_OUTPUT_NORM = auto()+ # qwen3tts code2wav: RVQ codes -> raw PCM+ A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)+ A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj+ A_GEN_WAV_QUANT_FIRST_CB = auto() # semantic RVQ codebook (1 layer), folded from embedding_sum/cluster_usage+ A_GEN_WAV_QUANT_REST_IN = auto() # acoustic RVQ, in_proj+ A_GEN_WAV_QUANT_REST_OUT = auto() # acoustic RVQ, out_proj+ A_GEN_WAV_QUANT_REST_CB = auto() # acoustic RVQ codebooks, merged 3D [15, vocab, dim]+ A_GEN_WAV_PRE_CONV = auto()+ A_GEN_WAV_TFM_IN_PROJ = auto()+ A_GEN_WAV_TFM_OUT_PROJ = auto()+ A_GEN_WAV_TFM_OUTPUT_NORM = auto()+ A_GEN_WAV_TFM_ATTN_NORM = auto()+ A_GEN_WAV_TFM_ATTN_Q = auto()+ A_GEN_WAV_TFM_ATTN_K = auto()+ A_GEN_WAV_TFM_ATTN_V = auto()+ A_GEN_WAV_TFM_ATTN_OUT = auto()+ A_GEN_WAV_TFM_ATTN_SCALE = auto() # layer scale (gamma) on the attn output+ A_GEN_WAV_TFM_FFN_NORM = auto()+ A_GEN_WAV_TFM_FFN_GATE = auto()+ A_GEN_WAV_TFM_FFN_UP = auto()+ A_GEN_WAV_TFM_FFN_DOWN = auto()+ A_GEN_WAV_TFM_FFN_SCALE = auto() # layer scale (gamma) on the FFN output+ A_GEN_WAV_UP_CONV = auto() # causal ConvTranspose1d, 2x upsample+ A_GEN_WAV_UP_DWCONV = auto() # ConvNeXt depthwise conv+ A_GEN_WAV_UP_NORM = auto() # ConvNeXt LayerNorm+ A_GEN_WAV_UP_PW1 = auto() # ConvNeXt pointwise conv 1 (expand)+ A_GEN_WAV_UP_PW2 = auto() # ConvNeXt pointwise conv 2 (project)+ A_GEN_WAV_UP_GAMMA = auto() # ConvNeXt layer scale+ A_GEN_WAV_DAC_ENTRY = auto() # DAC conv_pre+ A_GEN_WAV_DAC_UP_SNAKE = auto() # DAC per-block SnakeBeta before the upsample conv+ A_GEN_WAV_DAC_UP_CONV = auto() # DAC per-block causal ConvTranspose1d+ A_GEN_WAV_DAC_RES_ACT1 = auto() # DAC residual unit, SnakeBeta before conv1+ A_GEN_WAV_DAC_RES_CONV1 = auto() # DAC residual unit, dilated causal conv+ A_GEN_WAV_DAC_RES_ACT2 = auto() # DAC residual unit, SnakeBeta before conv2+ A_GEN_WAV_DAC_RES_CONV2 = auto() # DAC residual unit, pointwise causal conv+ A_GEN_WAV_DAC_POST_SNAKE = auto() # DAC final SnakeBeta+ A_GEN_WAV_DAC_POST_CONV = auto() # DAC conv_post -> 1-channel PCM+ # pocket-tts: SEANet encoder (speaker path) and decoder (a.gen.wav path)+ A_ENC_SEANET_CONV_IN = auto()+ A_ENC_SEANET_CONV_OUT = auto()+ A_ENC_SEANET_RES_CONV1 = auto() # residual unit, dilated conv+ A_ENC_SEANET_RES_CONV2 = auto() # residual unit, pointwise conv+ A_ENC_SEANET_SCALE_CONV = auto() # strided downsample conv+ A_ENC_ATTN_SCALE = auto() # layer scale (gamma) on the attn output+ A_ENC_FFN_SCALE_LS = auto() # layer scale (gamma) on the FFN output+ A_ENC_SPEAKER_PROJ = auto() # voice latent -> backbone embd+ A_GEN_FLOW_INPUT_PROJ = auto()+ A_GEN_FLOW_COND_EMBD = auto()+ A_GEN_FLOW_TIME_FREQS = auto() # timestep embedder, stored cos/sin frequencies+ A_GEN_FLOW_TIME_UP = auto()+ A_GEN_FLOW_TIME_DOWN = auto()+ A_GEN_FLOW_TIME_NORM = auto() # RMSNorm alpha+ A_GEN_FLOW_BLK_NORM = auto() # AdaLN res block, in_ln+ A_GEN_FLOW_BLK_UP = auto()+ A_GEN_FLOW_BLK_DOWN = auto()+ A_GEN_FLOW_BLK_ADA = auto() # AdaLN modulation, -> shift/scale/gate+ A_GEN_FLOW_FINAL_ADA = auto() # final layer AdaLN modulation, -> shift/scale+ A_GEN_FLOW_FINAL_PROJ = auto()+ A_GEN_OUT_EOS = auto() # end-of-speech head on the backbone hidden state+ A_GEN_INPUT_LINEAR = auto() # generated latent -> backbone embd+ A_GEN_EMB_MEAN = auto() # latent denormalization stats+ A_GEN_EMB_STD = auto()+ A_GEN_WAV_QUANT_OUT = auto() # DummyQuantizer output_proj, latent -> decoder dim+ A_GEN_WAV_UPSAMPLE = auto() # frame rate -> encoder frame rate, depthwise convtr+ A_GEN_WAV_SEANET_CONV_IN = auto()+ A_GEN_WAV_SEANET_CONV_OUT = auto() # -> 1-channel PCM+ A_GEN_WAV_SEANET_RES_CONV1 = auto()+ A_GEN_WAV_SEANET_RES_CONV2 = auto()+ A_GEN_WAV_SEANET_SCALE_CONV = auto() # strided upsample convtr+ A_MMPROJ = auto()+ A_MMPROJ_FC = auto()+ A_MM_NORM_PRE = auto()+ A_MM_NORM_MID = auto()+ A_MM_EMBEDDING = auto() # gemma3n+ A_MM_HARD_EMB_NORM = auto() # gemma3n+ A_MM_SOFT_EMB_NORM = auto() # gemma3n+ A_MM_INP_PROJ = auto() # gemma3n+ A_MM_CODE_EMBD = auto() # mimo: text-side RVQ code embedding table ("text codebook"), merged 3D [n_channels, vocab, dim]+ A_MM_LOCAL_ATTN_Q = auto() # mimo: input_local_transformer (LLM-side connector)+ A_MM_LOCAL_ATTN_K = auto()+ A_MM_LOCAL_ATTN_V = auto()+ A_MM_LOCAL_ATTN_OUT = auto()+ A_MM_LOCAL_FFN_GATE = auto()+ A_MM_LOCAL_FFN_UP = auto()+ A_MM_LOCAL_FFN_DOWN = auto()+ A_MM_LOCAL_LN1 = auto()+ A_MM_LOCAL_LN2 = auto()+ A_MM_LOCAL_NORM = auto() # final norm after all input_local_transformer layers+ A_PER_DIM_K_SCALE = auto() # gemma4+ A_PER_DIM_SCALE = auto() # gemma4+ # nextn/mtp+ NEXTN_PROJ_PRE = auto()+ NEXTN_PROJ_POST = auto()+ NEXTN_EH_PROJ = auto()+ NEXTN_EMBED_TOKENS = auto()+ NEXTN_ENORM = auto()+ NEXTN_HNORM = auto()+ NEXTN_SHARED_HEAD_HEAD = auto()+ NEXTN_SHARED_HEAD_NORM = auto()+ NEXTN_HC_HEAD_NORM = auto()+ NEXTN_HC_HEAD_DOWN = auto()+ NEXTN_HC_HEAD_UP = auto()+ # eagle3+ FC = auto() # feature fusion layer+ D2T = auto() # draft to target vocabulary mapping+ # dspark+ DSPARK_MARKOV_W1 = auto() # markov head: prev-token embed+ DSPARK_MARKOV_W2 = auto() # markov head: bias projection+ DSPARK_CONF_PROJ = auto() # confidence head+ DFLASH_ATTN_CONV_BASE = auto()+ DFLASH_ATTN_CONV_PROJ = auto()+ DFLASH_FFN_CONV_BASE = auto()+ DFLASH_FFN_CONV_PROJ = auto()+ DFLASH_SELECTOR_PREV = auto()+ DFLASH_SELECTOR_NEXT = auto()+ DFLASH_SELECTOR_HIDDEN = auto()+ # lfm2 audio+ A_ENC_NORM_CONV = auto()+ A_ENC_LINEAR_POS = auto()+ A_ENC_POS_BIAS_U = auto()+ A_ENC_POS_BIAS_V = auto()+ A_ENC_OUT = auto()+ A_ENC_CONV_DW = auto() # SSM conv+ A_ENC_CONV_NORM = auto() # SSM conv+ A_ENC_CONV_PW1 = auto()+ A_ENC_CONV_PW2 = auto()+ A_ENC_CONV_NORM_MEAN = auto() # parakeet+ A_ENC_CONV_NORM_VAR = auto() # parakeet+ A_ENC_MEL_FILTERS = auto() # parakeet+ A_ENC_WINDOW = auto() # parakeet+ A_CTC_OUT = auto()+ A_CTC_OUT_MID = auto()+ A_ENC_ATTN_REL_POS_EMB = auto()+ # audio qformer projector+ A_QF_PROJ_QUERY = auto()+ A_QF_PROJ_NORM = auto()+ A_QF_PROJ_LINEAR = auto()+ A_QF_SELF_ATTN_Q = auto()+ A_QF_SELF_ATTN_K = auto()+ A_QF_SELF_ATTN_V = auto()+ A_QF_SELF_ATTN_O = auto()+ A_QF_SELF_ATTN_NORM = auto()+ A_QF_CROSS_ATTN_Q = auto()+ A_QF_CROSS_ATTN_K = auto()+ A_QF_CROSS_ATTN_V = auto()+ A_QF_CROSS_ATTN_O = auto()+ A_QF_CROSS_ATTN_NORM = auto()+ A_QF_FFN_UP = auto()+ A_QF_FFN_DOWN = auto()+ A_QF_FFN_NORM = auto()+++MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {+ MODEL_ARCH.MMPROJ: "clip", # dummy arch for clip.cpp+ MODEL_ARCH.LLAMA: "llama",+ MODEL_ARCH.LLAMA4: "llama4",+ MODEL_ARCH.DECI: "deci",+ MODEL_ARCH.FALCON: "falcon",+ MODEL_ARCH.BAICHUAN: "baichuan",+ MODEL_ARCH.GROK: "grok",+ MODEL_ARCH.GPT2: "gpt2",+ MODEL_ARCH.GPTJ: "gptj",+ MODEL_ARCH.GPTNEOX: "gptneox",+ MODEL_ARCH.MPT: "mpt",+ MODEL_ARCH.STARCODER: "starcoder",+ MODEL_ARCH.REFACT: "refact",+ MODEL_ARCH.BERT: "bert",+ MODEL_ARCH.MODERN_BERT: "modern-bert",+ MODEL_ARCH.NOMIC_BERT: "nomic-bert",+ MODEL_ARCH.NOMIC_BERT_MOE: "nomic-bert-moe",+ MODEL_ARCH.NEO_BERT: "neo-bert",+ MODEL_ARCH.JINA_BERT_V2: "jina-bert-v2",+ MODEL_ARCH.JINA_BERT_V3: "jina-bert-v3",+ MODEL_ARCH.EUROBERT: "eurobert",+ MODEL_ARCH.BLOOM: "bloom",+ MODEL_ARCH.STABLELM: "stablelm",+ MODEL_ARCH.QWEN: "qwen",+ MODEL_ARCH.QWEN2: "qwen2",+ MODEL_ARCH.QWEN2MOE: "qwen2moe",+ MODEL_ARCH.QWEN2VL: "qwen2vl",+ MODEL_ARCH.QWEN3: "qwen3",+ MODEL_ARCH.QWEN3MOE: "qwen3moe",+ MODEL_ARCH.QWEN3NEXT: "qwen3next",+ MODEL_ARCH.HRM_TEXT: "hrm_text",+ MODEL_ARCH.QWEN3VL: "qwen3vl",+ MODEL_ARCH.QWEN3VLMOE: "qwen3vlmoe",+ MODEL_ARCH.QWEN35: "qwen35",+ MODEL_ARCH.QWEN35MOE: "qwen35moe",+ MODEL_ARCH.CLEF: "clef",+ MODEL_ARCH.VLLM_DECISION2: "vllm-decision2",+ MODEL_ARCH.QWEN4EXP: "qwen4exp",+ MODEL_ARCH.PHI2: "phi2",+ MODEL_ARCH.PHI3: "phi3",+ MODEL_ARCH.PHIMOE: "phimoe",+ MODEL_ARCH.PLAMO: "plamo",+ MODEL_ARCH.PLAMO2: "plamo2",+ MODEL_ARCH.PLAMO3: "plamo3",+ MODEL_ARCH.CODESHELL: "codeshell",+ MODEL_ARCH.ORION: "orion",+ MODEL_ARCH.INTERNLM2: "internlm2",+ MODEL_ARCH.MINICPM: "minicpm",+ MODEL_ARCH.MINICPM3: "minicpm3",+ MODEL_ARCH.GEMMA: "gemma",+ MODEL_ARCH.GEMMA2: "gemma2",+ MODEL_ARCH.GEMMA3: "gemma3",+ MODEL_ARCH.GEMMA3N: "gemma3n",+ MODEL_ARCH.GEMMA4: "gemma4",+ MODEL_ARCH.GEMMA4_ASSISTANT: "gemma4-assistant",+ MODEL_ARCH.GEMMA_EMBEDDING: "gemma-embedding",+ MODEL_ARCH.GEMMA_EMBEDDING2: "gemma-embedding2",+ MODEL_ARCH.STARCODER2: "starcoder2",+ MODEL_ARCH.RWKV6: "rwkv6",+ MODEL_ARCH.RWKV6QWEN2: "rwkv6qwen2",+ MODEL_ARCH.RWKV7: "rwkv7",+ MODEL_ARCH.ARWKV7: "arwkv7",+ MODEL_ARCH.MAMBA: "mamba",+ MODEL_ARCH.MAMBA2: "mamba2",+ MODEL_ARCH.MAPLE: "maple",+ MODEL_ARCH.JAMBA: "jamba",+ MODEL_ARCH.XVERSE: "xverse",+ MODEL_ARCH.COMMAND_R: "command-r",+ MODEL_ARCH.COHERE2: "cohere2",+ MODEL_ARCH.COHERE2MOE: "cohere2moe",+ MODEL_ARCH.DBRX: "dbrx",+ MODEL_ARCH.OLMO: "olmo",+ MODEL_ARCH.OLMO2: "olmo2",+ MODEL_ARCH.OLMOE: "olmoe",+ MODEL_ARCH.MUSE_GLIMMER: "muse-glimmer",+ MODEL_ARCH.OPENELM: "openelm",+ MODEL_ARCH.ARCTIC: "arctic",+ MODEL_ARCH.DEEPSEEK: "deepseek",+ MODEL_ARCH.DEEPSEEK2: "deepseek2",+ MODEL_ARCH.DEEPSEEK2OCR: "deepseek2-ocr",+ MODEL_ARCH.DEEPSEEK32: "deepseek32",+ MODEL_ARCH.DEEPSEEK4: "deepseek4",+ MODEL_ARCH.CHATGLM: "chatglm",+ MODEL_ARCH.GLM4: "glm4",+ MODEL_ARCH.GLM4_MOE: "glm4moe",+ MODEL_ARCH.GLM_DSA: "glm-dsa",+ MODEL_ARCH.GLM5_NEXT: "glm5-next",+ MODEL_ARCH.BITNET: "bitnet",+ MODEL_ARCH.T5: "t5",+ MODEL_ARCH.T5ENCODER: "t5encoder",+ MODEL_ARCH.JAIS: "jais",+ MODEL_ARCH.JAIS2: "jais2",+ MODEL_ARCH.NEMOTRON: "nemotron",+ MODEL_ARCH.NEMOTRON_H: "nemotron_h",+ MODEL_ARCH.NEMOTRON_H_MOE: "nemotron_h_moe",+ MODEL_ARCH.EXAONE: "exaone",+ MODEL_ARCH.EXAONE4: "exaone4",+ MODEL_ARCH.EXAONE_MOE: "exaone-moe",+ MODEL_ARCH.GRANITE: "granite",+ MODEL_ARCH.GRANITE_MOE: "granitemoe",+ MODEL_ARCH.GRANITE_HYBRID: "granitehybrid",+ MODEL_ARCH.GRANITE_SWITCH: "graniteswitch",+ MODEL_ARCH.GRANITE_SWA: "granite_swa",+ MODEL_ARCH.CHAMELEON: "chameleon",+ MODEL_ARCH.WAVTOKENIZER_DEC: "wavtokenizer-dec",+ MODEL_ARCH.PLM: "plm",+ MODEL_ARCH.BAILINGMOE: "bailingmoe",+ MODEL_ARCH.BAILINGMOE2: "bailingmoe2",+ MODEL_ARCH.BAILINGMOE3: "bailingmoe3",+ MODEL_ARCH.DOTS1: "dots1",+ MODEL_ARCH.DOTS3NOTE: "dots3note",+ MODEL_ARCH.ARCEE: "arcee",+ MODEL_ARCH.AFMOE: "afmoe",+ MODEL_ARCH.LAGUNA: "laguna",+ MODEL_ARCH.ERNIE4_5: "ernie4_5",+ MODEL_ARCH.ERNIE4_5_MOE: "ernie4_5-moe",+ MODEL_ARCH.FALCON_H1: "falcon-h1",+ MODEL_ARCH.HUNYUAN_MOE: "hunyuan-moe",+ MODEL_ARCH.HUNYUAN_DENSE: "hunyuan-dense",+ MODEL_ARCH.HUNYUAN_VL: "hunyuan_vl",+ MODEL_ARCH.HY_V3: "hy_v3",+ MODEL_ARCH.HY_V4: "hy_v4",+ MODEL_ARCH.SMOLLM3: "smollm3",+ MODEL_ARCH.GPT_OSS: "gpt-oss",+ MODEL_ARCH.LFM2: "lfm2",+ MODEL_ARCH.LFM2MOE: "lfm2moe",+ MODEL_ARCH.DREAM: "dream",+ MODEL_ARCH.SMALLTHINKER: "smallthinker",+ MODEL_ARCH.LLADA: "llada",+ MODEL_ARCH.LLADA_MOE: "llada-moe",+ MODEL_ARCH.SEED_OSS: "seed_oss",+ MODEL_ARCH.GROVEMOE: "grovemoe",+ MODEL_ARCH.APERTUS: "apertus",+ MODEL_ARCH.MINIMAX01: "minimax-01",+ MODEL_ARCH.MINIMAXM2: "minimax-m2",+ MODEL_ARCH.MINIMAXM3: "minimax-m3",+ MODEL_ARCH.COGVLM: "cogvlm",+ MODEL_ARCH.RND1: "rnd1",+ MODEL_ARCH.PANGU_EMBED: "pangu-embedded",+ MODEL_ARCH.MISTRAL3: "mistral3",+ MODEL_ARCH.EAGLE3: "eagle3",+ MODEL_ARCH.DFLASH: "dflash",+ MODEL_ARCH.MISTRAL4: "mistral4",+ MODEL_ARCH.PADDLEOCR: "paddleocr",+ MODEL_ARCH.MIMO2: "mimo2",+ MODEL_ARCH.STEP35: "step35",+ MODEL_ARCH.SPARK2_5: "spark2_5",+ MODEL_ARCH.LLAMA_EMBED: "llama-embed",+ MODEL_ARCH.MAINCODER: "maincoder",+ MODEL_ARCH.KIMI_LINEAR: "kimi-linear",+ MODEL_ARCH.KIMI_K3: "kimi-k3",+ MODEL_ARCH.TALKIE: "talkie",+ MODEL_ARCH.MELLUM: "mellum",+ MODEL_ARCH.NANBEIGE: "nanbeige",+ MODEL_ARCH.QWEN3TTS: "qwen3tts",+ MODEL_ARCH.POCKETTTS: "pockettts",+ MODEL_ARCH.K2HORIZON: "k2-horizon",+}++VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {+ VISION_PROJECTOR_TYPE.MLP: "mlp",+ VISION_PROJECTOR_TYPE.LDP: "ldp",+ VISION_PROJECTOR_TYPE.LDPV2: "ldpv2",+ VISION_PROJECTOR_TYPE.RESAMPLER: "resampler",+ VISION_PROJECTOR_TYPE.GLM_EDGE: "adapter",+ VISION_PROJECTOR_TYPE.MERGER: "qwen2vl_merger",+ VISION_PROJECTOR_TYPE.GEMMA3: "gemma3",+ VISION_PROJECTOR_TYPE.QWEN3VL: "qwen3vl_merger",+ VISION_PROJECTOR_TYPE.LING3VL: "ling3vl",+ VISION_PROJECTOR_TYPE.STEP3VL: "step3vl",+}++TENSOR_NAMES: dict[MODEL_TENSOR, str] = {+ MODEL_TENSOR.TOKEN_EMBD: "token_embd",+ MODEL_TENSOR.TOKEN_EMBD_NORM: "token_embd_norm",+ MODEL_TENSOR.TOKEN_TYPES: "token_types",+ MODEL_TENSOR.MASKED_EMBD_CENTROIDS: "masked_embd_centroids",+ MODEL_TENSOR.MASKED_EMBD_ORDERING: "masked_embd_ordering",+ MODEL_TENSOR.POS_EMBD: "position_embd",+ MODEL_TENSOR.OUTPUT_NORM: "output_norm",+ MODEL_TENSOR.OUTPUT: "output",+ MODEL_TENSOR.HRM_Z_L_INIT: "hrm.z_l_init",+ MODEL_TENSOR.DENSE_2_OUT: "dense_2", # embeddinggemma 2_Dense+ MODEL_TENSOR.DENSE_3_OUT: "dense_3", # embeddinggemma 2_Dense+ MODEL_TENSOR.HC_HEAD_FN: "output_hc_fn",+ MODEL_TENSOR.HC_HEAD_BASE: "output_hc_base",+ MODEL_TENSOR.HC_HEAD_SCALE: "output_hc_scale",+ MODEL_TENSOR.HC_HEAD_NORM: "output_hc_norm", # qwen4exp+ MODEL_TENSOR.HC_HEAD_DOWN: "output_hc_down", # qwen4exp+ MODEL_TENSOR.HC_HEAD_UP: "output_hc_up", # qwen4exp+ MODEL_TENSOR.ROPE_FREQS: "rope_freqs",+ MODEL_TENSOR.ROPE_FACTORS_LONG: "rope_factors_long",+ MODEL_TENSOR.ROPE_FACTORS_SHORT: "rope_factors_short",+ MODEL_TENSOR.ATTN_NORM: "blk.{bid}.attn_norm",+ MODEL_TENSOR.ATTN_NORM_2: "blk.{bid}.attn_norm_2",+ MODEL_TENSOR.ATTN_QKV: "blk.{bid}.attn_qkv",+ MODEL_TENSOR.ATTN_Q: "blk.{bid}.attn_q",+ MODEL_TENSOR.ATTN_K: "blk.{bid}.attn_k",+ MODEL_TENSOR.ATTN_V: "blk.{bid}.attn_v",+ MODEL_TENSOR.ATTN_OUT: "blk.{bid}.attn_output",+ MODEL_TENSOR.ATTN_ROT_EMBD: "blk.{bid}.attn_rot_embd",+ MODEL_TENSOR.ATTN_SINKS: "blk.{bid}.attn_sinks",+ MODEL_TENSOR.ATTN_GATE: "blk.{bid}.attn_gate",+ MODEL_TENSOR.ATTN_Q_NORM: "blk.{bid}.attn_q_norm",+ MODEL_TENSOR.ATTN_K_NORM: "blk.{bid}.attn_k_norm",+ MODEL_TENSOR.ATTN_OUT_NORM: "blk.{bid}.attn_output_norm",+ MODEL_TENSOR.ATTN_POST_NORM: "blk.{bid}.post_attention_norm",+ MODEL_TENSOR.FFN_GATE_INP: "blk.{bid}.ffn_gate_inp",+ MODEL_TENSOR.FFN_GATE_INP_SHEXP: "blk.{bid}.ffn_gate_inp_shexp",+ MODEL_TENSOR.FFN_NORM: "blk.{bid}.ffn_norm",+ MODEL_TENSOR.FFN_PRE_NORM: "blk.{bid}.ffn_norm",+ MODEL_TENSOR.FFN_POST_NORM: "blk.{bid}.post_ffw_norm",+ MODEL_TENSOR.FFN_PRE_NORM_2: "blk.{bid}.pre_ffw_norm_2", # gemma4+ MODEL_TENSOR.FFN_POST_NORM_1: "blk.{bid}.post_ffw_norm_1", # gemma4+ MODEL_TENSOR.FFN_POST_NORM_2: "blk.{bid}.post_ffw_norm_2", # gemma4+ MODEL_TENSOR.FFN_GATE: "blk.{bid}.ffn_gate",+ MODEL_TENSOR.FFN_DOWN: "blk.{bid}.ffn_down",+ MODEL_TENSOR.FFN_UP: "blk.{bid}.ffn_up",+ MODEL_TENSOR.FFN_GATE_SHEXP: "blk.{bid}.ffn_gate_shexp",+ MODEL_TENSOR.FFN_DOWN_SHEXP: "blk.{bid}.ffn_down_shexp",+ MODEL_TENSOR.FFN_UP_SHEXP: "blk.{bid}.ffn_up_shexp",+ MODEL_TENSOR.FFN_GATE_CHEXP: "blk.{bid}.ffn_gate_chexps",+ MODEL_TENSOR.FFN_DOWN_CHEXP: "blk.{bid}.ffn_down_chexps",+ MODEL_TENSOR.FFN_UP_CHEXP: "blk.{bid}.ffn_up_chexps",+ MODEL_TENSOR.FFN_ACT: "blk.{bid}.ffn",+ MODEL_TENSOR.FFN_NORM_EXP: "blk.{bid}.ffn_norm_exps",+ MODEL_TENSOR.FFN_GATE_EXP: "blk.{bid}.ffn_gate_exps",+ MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down_exps",+ MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps",+ MODEL_TENSOR.FFN_GATE_UP_EXP: "blk.{bid}.ffn_gate_up_exps",+ MODEL_TENSOR.FFN_EXP_PROBS_B: "blk.{bid}.exp_probs_b",+ MODEL_TENSOR.FFN_EXP_PROBS_B_VL: "blk.{bid}.exp_probs_b_vl",+ MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid",+ MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super+ MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super+ MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm",+ MODEL_TENSOR.LAYER_OUT_SCALE: "blk.{bid}.layer_output_scale",+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD: "per_layer_token_embd", # gemma3n+ MODEL_TENSOR.PER_LAYER_MODEL_PROJ: "per_layer_model_proj", # gemma3n+ MODEL_TENSOR.PER_LAYER_PROJ_NORM: "per_layer_proj_norm", # gemma3n+ MODEL_TENSOR.ALTUP_UNEMBD_PROJ: "altup_unembd_proj", # gemma3n+ MODEL_TENSOR.ALTUP_PROJ: "altup_proj", # gemma3n+ MODEL_TENSOR.PER_LAYER_INP_GATE: "blk.{bid}.inp_gate", # gemma3n+ MODEL_TENSOR.PER_LAYER_PROJ: "blk.{bid}.proj", # gemma3n+ MODEL_TENSOR.PER_LAYER_POST_NORM: "blk.{bid}.post_norm", # gemma3n+ MODEL_TENSOR.ALTUP_CORRECT_COEF: "blk.{bid}.altup_correct_coef", # gemma3n+ MODEL_TENSOR.ALTUP_CORRECT_SCALE: "blk.{bid}.altup_correct_scale", # gemma3n+ MODEL_TENSOR.ALTUP_PREDICT_COEF: "blk.{bid}.altup_predict_coef", # gemma3n+ MODEL_TENSOR.ALTUP_ROUTER: "blk.{bid}.altup_router", # gemma3n+ MODEL_TENSOR.ALTUP_ROUTER_NORM: "blk.{bid}.altup_router_norm", # gemma3n+ MODEL_TENSOR.LAUREL_L: "blk.{bid}.laurel_l", # gemma3n+ MODEL_TENSOR.LAUREL_R: "blk.{bid}.laurel_r", # gemma3n+ MODEL_TENSOR.LAUREL_POST_NORM: "blk.{bid}.laurel_post_norm", # gemma3n+ MODEL_TENSOR.SSM_IN: "blk.{bid}.ssm_in",+ MODEL_TENSOR.SSM_CONV1D: "blk.{bid}.ssm_conv1d",+ MODEL_TENSOR.SSM_X: "blk.{bid}.ssm_x",+ MODEL_TENSOR.SSM_DT: "blk.{bid}.ssm_dt",+ MODEL_TENSOR.SSM_DT_NORM: "blk.{bid}.ssm_dt_norm",+ MODEL_TENSOR.SSM_A: "blk.{bid}.ssm_a",+ MODEL_TENSOR.SSM_B_NORM: "blk.{bid}.ssm_b_norm",+ MODEL_TENSOR.SSM_C_NORM: "blk.{bid}.ssm_c_norm",+ MODEL_TENSOR.SSM_D: "blk.{bid}.ssm_d",+ MODEL_TENSOR.SSM_NORM: "blk.{bid}.ssm_norm",+ MODEL_TENSOR.SSM_OUT: "blk.{bid}.ssm_out",+ MODEL_TENSOR.SSM_ALPHA: "blk.{bid}.ssm_alpha", # qwen3.5+ MODEL_TENSOR.SSM_BETA_ALPHA: "blk.{bid}.ssm_ba",+ MODEL_TENSOR.SSM_CONV1D_Q: "blk.{bid}.ssm_conv1d_q", # Kimi Linear+ MODEL_TENSOR.SSM_CONV1D_K: "blk.{bid}.ssm_conv1d_k", # Kimi Linear+ MODEL_TENSOR.SSM_CONV1D_V: "blk.{bid}.ssm_conv1d_v", # Kimi Linear+ MODEL_TENSOR.SSM_F_A: "blk.{bid}.ssm_f_a", # Kimi Linear+ MODEL_TENSOR.SSM_F_B: "blk.{bid}.ssm_f_b", # Kimi Linear+ MODEL_TENSOR.SSM_BETA: "blk.{bid}.ssm_beta", # Kimi Linear qwen3.5+ MODEL_TENSOR.SSM_G_A: "blk.{bid}.ssm_g_a", # Kimi Linear+ MODEL_TENSOR.SSM_G_B: "blk.{bid}.ssm_g_b", # Kimi Linear+ MODEL_TENSOR.SSM_G: "blk.{bid}.ssm_g", # Kimi K3+ MODEL_TENSOR.ATTN_RES_SCORE: "blk.{bid}.attn_res_score", # Kimi K3+ MODEL_TENSOR.FFN_RES_SCORE: "blk.{bid}.ffn_res_score", # Kimi K3+ MODEL_TENSOR.OUTPUT_RES_SCORE: "output_res_score", # Kimi K3+ MODEL_TENSOR.FFN_ROUTED_DOWN: "blk.{bid}.ffn_routed_down", # Kimi K3+ MODEL_TENSOR.FFN_ROUTED_UP: "blk.{bid}.ffn_routed_up", # Kimi K3+ MODEL_TENSOR.FFN_ROUTED_NORM: "blk.{bid}.ffn_routed_norm", # Kimi K3+ MODEL_TENSOR.TIME_MIX_W0: "blk.{bid}.time_mix_w0",+ MODEL_TENSOR.TIME_MIX_W1: "blk.{bid}.time_mix_w1",+ MODEL_TENSOR.TIME_MIX_W2: "blk.{bid}.time_mix_w2",+ MODEL_TENSOR.TIME_MIX_A0: "blk.{bid}.time_mix_a0",+ MODEL_TENSOR.TIME_MIX_A1: "blk.{bid}.time_mix_a1",+ MODEL_TENSOR.TIME_MIX_A2: "blk.{bid}.time_mix_a2",+ MODEL_TENSOR.TIME_MIX_V0: "blk.{bid}.time_mix_v0",+ MODEL_TENSOR.TIME_MIX_V1: "blk.{bid}.time_mix_v1",+ MODEL_TENSOR.TIME_MIX_V2: "blk.{bid}.time_mix_v2",+ MODEL_TENSOR.TIME_MIX_G1: "blk.{bid}.time_mix_g1",+ MODEL_TENSOR.TIME_MIX_G2: "blk.{bid}.time_mix_g2",+ MODEL_TENSOR.TIME_MIX_K_K: "blk.{bid}.time_mix_k_k",+ MODEL_TENSOR.TIME_MIX_K_A: "blk.{bid}.time_mix_k_a",+ MODEL_TENSOR.TIME_MIX_R_K: "blk.{bid}.time_mix_r_k",+ MODEL_TENSOR.TIME_MIX_LERP_X: "blk.{bid}.time_mix_lerp_x",+ MODEL_TENSOR.TIME_MIX_LERP_K: "blk.{bid}.time_mix_lerp_k",+ MODEL_TENSOR.TIME_MIX_LERP_V: "blk.{bid}.time_mix_lerp_v",+ MODEL_TENSOR.TIME_MIX_LERP_R: "blk.{bid}.time_mix_lerp_r",+ MODEL_TENSOR.TIME_MIX_LERP_G: "blk.{bid}.time_mix_lerp_g",+ MODEL_TENSOR.TIME_MIX_LERP_FUSED: "blk.{bid}.time_mix_lerp_fused",+ MODEL_TENSOR.TIME_MIX_LERP_W: "blk.{bid}.time_mix_lerp_w",+ MODEL_TENSOR.TIME_MIX_FIRST: "blk.{bid}.time_mix_first",+ MODEL_TENSOR.TIME_MIX_DECAY: "blk.{bid}.time_mix_decay",+ MODEL_TENSOR.TIME_MIX_DECAY_W1: "blk.{bid}.time_mix_decay_w1",+ MODEL_TENSOR.TIME_MIX_DECAY_W2: "blk.{bid}.time_mix_decay_w2",+ MODEL_TENSOR.TIME_MIX_KEY: "blk.{bid}.time_mix_key",+ MODEL_TENSOR.TIME_MIX_VALUE: "blk.{bid}.time_mix_value",+ MODEL_TENSOR.TIME_MIX_RECEPTANCE: "blk.{bid}.time_mix_receptance",+ MODEL_TENSOR.TIME_MIX_GATE: "blk.{bid}.time_mix_gate",+ MODEL_TENSOR.TIME_MIX_LN: "blk.{bid}.time_mix_ln",+ MODEL_TENSOR.TIME_MIX_OUTPUT: "blk.{bid}.time_mix_output",+ MODEL_TENSOR.CHANNEL_MIX_LERP_K: "blk.{bid}.channel_mix_lerp_k",+ MODEL_TENSOR.CHANNEL_MIX_LERP_R: "blk.{bid}.channel_mix_lerp_r",+ MODEL_TENSOR.CHANNEL_MIX_KEY: "blk.{bid}.channel_mix_key",+ MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE: "blk.{bid}.channel_mix_receptance",+ MODEL_TENSOR.CHANNEL_MIX_VALUE: "blk.{bid}.channel_mix_value",+ MODEL_TENSOR.ATTN_Q_A: "blk.{bid}.attn_q_a",+ MODEL_TENSOR.ATTN_Q_B: "blk.{bid}.attn_q_b",+ MODEL_TENSOR.ATTN_KV_A_MQA: "blk.{bid}.attn_kv_a_mqa",+ MODEL_TENSOR.ATTN_KV_B: "blk.{bid}.attn_kv_b",+ MODEL_TENSOR.ATTN_K_B: "blk.{bid}.attn_k_b",+ MODEL_TENSOR.ATTN_V_B: "blk.{bid}.attn_v_b",+ MODEL_TENSOR.ATTN_Q_A_NORM: "blk.{bid}.attn_q_a_norm",+ MODEL_TENSOR.ATTN_KV_A_NORM: "blk.{bid}.attn_kv_a_norm",+ MODEL_TENSOR.ATTN_KV: "blk.{bid}.attn_kv",+ MODEL_TENSOR.ATTN_KV_NORM: "blk.{bid}.attn_kv_a_norm",+ MODEL_TENSOR.ATTN_OUT_A: "blk.{bid}.attn_output_a",+ MODEL_TENSOR.ATTN_OUT_B: "blk.{bid}.attn_output_b",+ MODEL_TENSOR.HC_ATTN_FN: "blk.{bid}.hc_attn_fn",+ MODEL_TENSOR.HC_ATTN_BASE: "blk.{bid}.hc_attn_base",+ MODEL_TENSOR.HC_ATTN_SCALE: "blk.{bid}.hc_attn_scale",+ MODEL_TENSOR.HC_FFN_FN: "blk.{bid}.hc_ffn_fn",+ MODEL_TENSOR.HC_FFN_BASE: "blk.{bid}.hc_ffn_base",+ MODEL_TENSOR.HC_FFN_SCALE: "blk.{bid}.hc_ffn_scale",+ MODEL_TENSOR.HC_ATTN_NORM: "blk.{bid}.hc_attn_norm", # qwen4exp+ MODEL_TENSOR.HC_ATTN_DOWN: "blk.{bid}.hc_attn_down", # qwen4exp+ MODEL_TENSOR.HC_ATTN_UP: "blk.{bid}.hc_attn_up", # qwen4exp+ MODEL_TENSOR.HC_ATTN_INJECT: "blk.{bid}.hc_attn_inject", # qwen4exp+ MODEL_TENSOR.HC_FFN_NORM: "blk.{bid}.hc_ffn_norm", # qwen4exp+ MODEL_TENSOR.HC_FFN_DOWN: "blk.{bid}.hc_ffn_down", # qwen4exp+ MODEL_TENSOR.HC_FFN_UP: "blk.{bid}.hc_ffn_up", # qwen4exp+ MODEL_TENSOR.HC_FFN_INJECT: "blk.{bid}.hc_ffn_inject", # qwen4exp+ MODEL_TENSOR.PLE_KEY: "blk.{bid}.ple_key", # qwen4exp+ MODEL_TENSOR.PLE_VALUE: "blk.{bid}.ple_value", # qwen4exp+ MODEL_TENSOR.PLE_NORM_KEY: "blk.{bid}.ple_norm_key", # qwen4exp+ MODEL_TENSOR.PLE_NORM_QUERY: "blk.{bid}.ple_norm_query", # qwen4exp+ MODEL_TENSOR.PLE_NORM_CONV: "blk.{bid}.ple_norm_conv", # qwen4exp+ MODEL_TENSOR.PLE_CONV1D: "blk.{bid}.ple_conv1d", # qwen4exp+ MODEL_TENSOR.ATTN_COMPRESSOR_WKV: "blk.{bid}.attn_compressor_kv",+ MODEL_TENSOR.ATTN_COMPRESSOR_WGATE: "blk.{bid}.attn_compressor_gate",+ MODEL_TENSOR.ATTN_COMPRESSOR_APE: "blk.{bid}.attn_compressor_ape",+ MODEL_TENSOR.ATTN_COMPRESSOR_NORM: "blk.{bid}.attn_compressor_norm",+ MODEL_TENSOR.ATTN_SUB_NORM: "blk.{bid}.attn_sub_norm",+ MODEL_TENSOR.FFN_SUB_NORM: "blk.{bid}.ffn_sub_norm",+ MODEL_TENSOR.DEC_ATTN_NORM: "dec.blk.{bid}.attn_norm",+ MODEL_TENSOR.DEC_ATTN_Q: "dec.blk.{bid}.attn_q",+ MODEL_TENSOR.DEC_ATTN_K: "dec.blk.{bid}.attn_k",+ MODEL_TENSOR.DEC_ATTN_V: "dec.blk.{bid}.attn_v",+ MODEL_TENSOR.DEC_ATTN_OUT: "dec.blk.{bid}.attn_o",+ MODEL_TENSOR.DEC_ATTN_REL_B: "dec.blk.{bid}.attn_rel_b",+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM: "dec.blk.{bid}.cross_attn_norm",+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV: "dec.blk.{bid}.cross_attn_norm_kv",+ MODEL_TENSOR.DEC_CROSS_ATTN_Q: "dec.blk.{bid}.cross_attn_q",+ MODEL_TENSOR.DEC_CROSS_ATTN_K: "dec.blk.{bid}.cross_attn_k",+ MODEL_TENSOR.DEC_CROSS_ATTN_V: "dec.blk.{bid}.cross_attn_v",+ MODEL_TENSOR.DEC_CROSS_ATTN_OUT: "dec.blk.{bid}.cross_attn_o",+ MODEL_TENSOR.DEC_CROSS_ATTN_REL_B: "dec.blk.{bid}.cross_attn_rel_b",+ MODEL_TENSOR.DEC_FFN_NORM: "dec.blk.{bid}.ffn_norm",+ MODEL_TENSOR.DEC_FFN_GATE: "dec.blk.{bid}.ffn_gate",+ MODEL_TENSOR.DEC_FFN_DOWN: "dec.blk.{bid}.ffn_down",+ MODEL_TENSOR.DEC_FFN_UP: "dec.blk.{bid}.ffn_up",+ MODEL_TENSOR.DEC_OUTPUT_NORM: "dec.output_norm",+ MODEL_TENSOR.ENC_ATTN_NORM: "enc.blk.{bid}.attn_norm",+ MODEL_TENSOR.ENC_ATTN_Q: "enc.blk.{bid}.attn_q",+ MODEL_TENSOR.ENC_ATTN_K: "enc.blk.{bid}.attn_k",+ MODEL_TENSOR.ENC_ATTN_V: "enc.blk.{bid}.attn_v",+ MODEL_TENSOR.ENC_ATTN_OUT: "enc.blk.{bid}.attn_o",+ MODEL_TENSOR.ENC_ATTN_REL_B: "enc.blk.{bid}.attn_rel_b",+ MODEL_TENSOR.ENC_FFN_NORM: "enc.blk.{bid}.ffn_norm",+ MODEL_TENSOR.ENC_FFN_GATE: "enc.blk.{bid}.ffn_gate",+ MODEL_TENSOR.ENC_FFN_DOWN: "enc.blk.{bid}.ffn_down",+ MODEL_TENSOR.ENC_FFN_UP: "enc.blk.{bid}.ffn_up",+ MODEL_TENSOR.ENC_OUTPUT_NORM: "enc.output_norm",+ MODEL_TENSOR.CLS: "cls",+ MODEL_TENSOR.CLS_OUT: "cls.output",+ MODEL_TENSOR.CLS_NORM: "cls.norm",+ MODEL_TENSOR.DECISION_HIDDEN_NORM: "decision.hidden_norm",+ MODEL_TENSOR.DECISION_PROJ_MEMORY: "decision.proj_memory",+ MODEL_TENSOR.DECISION_PROJ_QUESTION: "decision.proj_question",+ MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION: "decision.proj_option_question",+ MODEL_TENSOR.DECISION_PROJ_GLOBAL: "decision.proj_global",+ MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT: "decision.proj_option_context",+ MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL: "decision.proj_option_lexical",+ MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM: "decision.option_summary_norm",+ MODEL_TENSOR.DECISION_FIELD_NORM: "decision.field_norm",+ MODEL_TENSOR.DECISION_OPTION_NORM: "decision.option_norm",+ MODEL_TENSOR.DECISION_SCALES: "decision.scales",+ MODEL_TENSOR.DECISION_SCORER: "decision.scorer",+ MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out",+ MODEL_TENSOR.DECISION_QUERY_NORM: "decision.query_norm",+ MODEL_TENSOR.DECISION_KEY: "decision.key",+ MODEL_TENSOR.DECISION_QUERY: "decision.query",+ MODEL_TENSOR.DECISION_OPTION_MLP: "decision.option_mlp",+ MODEL_TENSOR.DECISION_QUERY_MLP: "decision.query_mlp",+ MODEL_TENSOR.CONV1D: "conv1d",+ MODEL_TENSOR.CONVNEXT_DW: "convnext.{bid}.dw",+ MODEL_TENSOR.CONVNEXT_NORM: "convnext.{bid}.norm",+ MODEL_TENSOR.CONVNEXT_PW1: "convnext.{bid}.pw1",+ MODEL_TENSOR.CONVNEXT_PW2: "convnext.{bid}.pw2",+ MODEL_TENSOR.CONVNEXT_GAMMA: "convnext.{bid}.gamma",+ MODEL_TENSOR.POSNET_CONV1: "posnet.{bid}.conv1",+ MODEL_TENSOR.POSNET_CONV2: "posnet.{bid}.conv2",+ MODEL_TENSOR.POSNET_NORM: "posnet.{bid}.norm",+ MODEL_TENSOR.POSNET_NORM1: "posnet.{bid}.norm1",+ MODEL_TENSOR.POSNET_NORM2: "posnet.{bid}.norm2",+ MODEL_TENSOR.POSNET_ATTN_NORM: "posnet.{bid}.attn_norm",+ MODEL_TENSOR.POSNET_ATTN_Q: "posnet.{bid}.attn_q",+ MODEL_TENSOR.POSNET_ATTN_K: "posnet.{bid}.attn_k",+ MODEL_TENSOR.POSNET_ATTN_V: "posnet.{bid}.attn_v",+ MODEL_TENSOR.POSNET_ATTN_OUT: "posnet.{bid}.attn_output",+ MODEL_TENSOR.SHORTCONV_CONV: "blk.{bid}.shortconv.conv",+ MODEL_TENSOR.SHORTCONV_INPROJ: "blk.{bid}.shortconv.in_proj",+ MODEL_TENSOR.SHORTCONV_OUTPROJ: "blk.{bid}.shortconv.out_proj",+ MODEL_TENSOR.VISEXP_ATTN_QKV: "blk.{bid}.vis_attn_qkv",+ MODEL_TENSOR.VISEXP_ATTN_OUT: "blk.{bid}.vis_attn_output",+ MODEL_TENSOR.VISEXP_GATE: "blk.{bid}.vis_gate",+ MODEL_TENSOR.VISEXP_DOWN: "blk.{bid}.vis_down",+ MODEL_TENSOR.VISEXP_UP: "blk.{bid}.vis_up",+ MODEL_TENSOR.INDEXER_K_NORM: "blk.{bid}.indexer.k_norm",+ MODEL_TENSOR.INDEXER_PROJ: "blk.{bid}.indexer.proj",+ MODEL_TENSOR.INDEXER_ATTN_K: "blk.{bid}.indexer.attn_k",+ MODEL_TENSOR.INDEXER_ATTN_Q_B: "blk.{bid}.indexer.attn_q_b",+ MODEL_TENSOR.INDEXER_Q_PROJ: "blk.{bid}.indexer.q_proj",+ MODEL_TENSOR.INDEXER_K_PROJ: "blk.{bid}.indexer.k_proj",+ MODEL_TENSOR.INDEXER_Q_NORM: "blk.{bid}.indexer.q_norm",+ MODEL_TENSOR.INDEXER_COMPRESSOR_WKV: "blk.{bid}.indexer_compressor_kv",+ MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE: "blk.{bid}.indexer_compressor_gate",+ MODEL_TENSOR.INDEXER_COMPRESSOR_APE: "blk.{bid}.indexer_compressor_ape",+ MODEL_TENSOR.INDEXER_COMPRESSOR_NORM: "blk.{bid}.indexer_compressor_norm",+ MODEL_TENSOR.INDEXER_KPOOL_GATE: "blk.{bid}.indexer_compressor_gate",+ MODEL_TENSOR.INDEXER_KPOOL_APE: "blk.{bid}.indexer_compressor_ape",+ # vision+ MODEL_TENSOR.V_MMPROJ: "mm.{bid}",+ MODEL_TENSOR.V_MMPROJ_FC: "mm.model.fc",+ MODEL_TENSOR.V_MMPROJ_MLP: "mm.model.mlp.{bid}",+ MODEL_TENSOR.V_MMPROJ_PEG: "mm.model.peg.{bid}",+ MODEL_TENSOR.V_ENC_EMBD_CLS: "v.class_embd",+ MODEL_TENSOR.V_ENC_EMBD_PATCH: "v.patch_embd",+ MODEL_TENSOR.V_ENC_EMBD_NORM: "v.norm_embd",+ MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: "v.patch_norm.{bid}",+ MODEL_TENSOR.V_ENC_EMBD_POS: "v.position_embd",+ MODEL_TENSOR.V_ENC_ATTN_QKV: "v.blk.{bid}.attn_qkv",+ MODEL_TENSOR.V_ENC_ATTN_Q: "v.blk.{bid}.attn_q",+ MODEL_TENSOR.V_ENC_ATTN_Q_NORM: "v.blk.{bid}.attn_q_norm",+ MODEL_TENSOR.V_ENC_ATTN_K: "v.blk.{bid}.attn_k",+ MODEL_TENSOR.V_ENC_ATTN_K_NORM: "v.blk.{bid}.attn_k_norm",+ MODEL_TENSOR.V_ENC_ATTN_V: "v.blk.{bid}.attn_v",+ MODEL_TENSOR.V_ENC_INPUT_NORM: "v.blk.{bid}.ln1",+ MODEL_TENSOR.V_ENC_ATTN_O: "v.blk.{bid}.attn_out",+ MODEL_TENSOR.V_ENC_ATTN_O_NORM: "v.blk.{bid}.attn_out_norm",+ MODEL_TENSOR.V_ENC_ATTN_SINKS: "v.blk.{bid}.attn_sinks",+ MODEL_TENSOR.V_ENC_POST_ATTN_NORM: "v.blk.{bid}.ln2",+ MODEL_TENSOR.V_ENC_FFN_UP: "v.blk.{bid}.ffn_up",+ MODEL_TENSOR.V_ENC_FFN_GATE: "v.blk.{bid}.ffn_gate",+ MODEL_TENSOR.V_ENC_FFN_DOWN: "v.blk.{bid}.ffn_down",+ MODEL_TENSOR.V_ENC_FFN_GATE_INP: "v.blk.{bid}.ffn_gate_inp",+ MODEL_TENSOR.V_ENC_FFN_GATE_EXPS: "v.blk.{bid}.ffn_gate_exps",+ MODEL_TENSOR.V_ENC_FFN_UP_EXPS: "v.blk.{bid}.ffn_up_exps",+ MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS: "v.blk.{bid}.ffn_down_exps",+ MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B: "v.blk.{bid}.exp_probs_b",+ MODEL_TENSOR.V_ENC_ATTN_POST_NORM: "v.blk.{bid}.attn_post_norm",+ MODEL_TENSOR.V_ENC_FFN_POST_NORM: "v.blk.{bid}.ffn_post_norm",+ MODEL_TENSOR.V_LAYER_SCALE_1: "v.blk.{bid}.ls1",+ MODEL_TENSOR.V_LAYER_SCALE_2: "v.blk.{bid}.ls2",+ MODEL_TENSOR.V_LAYER_OUT_SCALE: "v.blk.{bid}.out_scale",+ MODEL_TENSOR.V_PRE_NORM: "v.pre_ln",+ MODEL_TENSOR.V_POST_NORM: "v.post_ln",+ MODEL_TENSOR.V_MM_POST_NORM: "mm.post_norm",+ MODEL_TENSOR.V_MM_INP_PROJ: "mm.input_projection",+ MODEL_TENSOR.V_MM_INP_NORM: "mm.input_norm",+ MODEL_TENSOR.V_MM_SOFT_EMB_NORM: "mm.soft_emb_norm", # gemma3n+ MODEL_TENSOR.V_MM_EMBEDDING: "mm.embedding", # gemma3n+ MODEL_TENSOR.V_MM_HARD_EMB_NORM: "mm.hard_emb_norm", # gemma3n+ MODEL_TENSOR.V_ENC_CONV_STEM: "v.conv_stem.conv", # gemma3n+ MODEL_TENSOR.V_ENC_CONV_STEM_NORM: "v.conv_stem.bn", # gemma3n+ MODEL_TENSOR.V_ENC_MSFA_EXP: "v.msfa.ffn.pw_exp.conv", # gemma3n+ MODEL_TENSOR.V_ENC_MSFA_EXP_NORM: "v.msfa.ffn.pw_exp.bn", # gemma3n+ MODEL_TENSOR.V_ENC_MSFA_PROJ: "v.msfa.ffn.pw_proj.conv", # gemma3n+ MODEL_TENSOR.V_ENC_MSFA_PROJ_NORM: "v.msfa.ffn.pw_proj.bn", # gemma3n+ MODEL_TENSOR.V_ENC_MSFA_NORM: "v.msfa.norm", # gemma3n+ MODEL_TENSOR.V_RESMPL_POS_EMBD_K: "resampler.pos_embd_k",+ MODEL_TENSOR.V_RESMPL_ATTN_Q: "resampler.attn.q",+ MODEL_TENSOR.V_RESMPL_ATTN_K: "resampler.attn.k",+ MODEL_TENSOR.V_RESMPL_ATTN_V: "resampler.attn.v",+ MODEL_TENSOR.V_RESMPL_ATTN_OUT: "resampler.attn.out",+ MODEL_TENSOR.V_RESMPL_KV: "resampler.kv",+ MODEL_TENSOR.V_RESMPL_KV_NORM: "resampler.ln_kv",+ MODEL_TENSOR.V_RESMPL_POST_NORM: "resampler.ln_post",+ MODEL_TENSOR.V_RESMPL_Q_NORM: "resampler.ln_q",+ MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj",+ MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query",+ MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral+ MODEL_TENSOR.V_TOK_EMBD_IMG_START: "v.token_embd.img_start", # deepseek4v+ MODEL_TENSOR.V_TOK_EMBD_IMG_END: "v.token_embd.img_end", # deepseek4v+ MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: "v.token_embd.img_pad", # deepseek4v+ MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1+ MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm",+ MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1",+ MODEL_TENSOR.V_DS_FC2: "v.deepstack.{bid}.fc2",+ MODEL_TENSOR.V_MERGER_LN1: "v.vit_merger.ln1",+ MODEL_TENSOR.V_MERGER_ATTN_Q: "v.vit_merger.attn_q",+ MODEL_TENSOR.V_MERGER_ATTN_K: "v.vit_merger.attn_k",+ MODEL_TENSOR.V_MERGER_ATTN_V: "v.vit_merger.attn_v",+ MODEL_TENSOR.V_MERGER_ATTN_O: "v.vit_merger.attn_out",+ MODEL_TENSOR.V_MERGER_DS_LN: "v.vit_merger.ds_ln",+ MODEL_TENSOR.V_MERGER_DS_UP: "v.vit_merger.ds_ffn_up",+ MODEL_TENSOR.V_MERGER_DS_DOWN: "v.vit_merger.ds_ffn_down",+ MODEL_TENSOR.V_MM_POST_FC_NORM: "mm.post_fc_norm", # cogvlm+ MODEL_TENSOR.V_MM_UP: "mm.up",+ MODEL_TENSOR.V_MM_DOWN: "mm.down",+ MODEL_TENSOR.V_MM_GATE: "mm.gate",+ MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1",+ MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2",+ MODEL_TENSOR.V_TOK_BOI: "v.boi",+ MODEL_TENSOR.V_TOK_EOI: "v.eoi",+ MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm",+ MODEL_TENSOR.V_TOK_IMG_BEGIN: "mm.image_begin",+ MODEL_TENSOR.V_TOK_IMG_END: "mm.image_end",+ MODEL_TENSOR.V_STD_BIAS: "v.std_bias", # gemma4+ MODEL_TENSOR.V_STD_SCALE: "v.std_scale", # gemma4+ # DeepSeek-OCR SAM+ MODEL_TENSOR.V_SAM_POS_EMBD: "v.sam.pos_embd",+ MODEL_TENSOR.V_SAM_PATCH_EMBD: "v.sam.patch_embd",+ MODEL_TENSOR.V_SAM_PRE_NORM: "v.sam.blk.{bid}.pre_ln",+ MODEL_TENSOR.V_SAM_POST_NORM: "v.sam.blk.{bid}.post_ln",+ MODEL_TENSOR.V_SAM_ATTN_POS_H: "v.sam.blk.{bid}.attn.pos_h",+ MODEL_TENSOR.V_SAM_ATTN_POS_W: "v.sam.blk.{bid}.attn.pos_w",+ MODEL_TENSOR.V_SAM_ATTN_QKV: "v.sam.blk.{bid}.attn.qkv",+ MODEL_TENSOR.V_SAM_ATTN_OUT: "v.sam.blk.{bid}.attn.out",+ MODEL_TENSOR.V_SAM_MLP_LIN_1: "v.sam.blk.{bid}.mlp.lin1",+ MODEL_TENSOR.V_SAM_MLP_LIN_2: "v.sam.blk.{bid}.mlp.lin2",+ MODEL_TENSOR.V_SAM_NECK: "v.sam.neck.{bid}",+ MODEL_TENSOR.V_SAM_NET_2: "v.sam.net_2",+ MODEL_TENSOR.V_SAM_NET_3: "v.sam.net_3",+ MODEL_TENSOR.V_ENC_EMBD_IMGNL: "v.image_newline", # Deepseek-OCR, Granite4Vision+ MODEL_TENSOR.V_ENC_EMBD_VSEP: "v.view_seperator", # Deepseek-OCR+ MODEL_TENSOR.V_TOK_EMBD_SEP: "v.tok_embd_sep", # MiniCPM-V 4.7+ MODEL_TENSOR.V_RESMPL_QUERY_768: "v.resample_query_768", # Deepseek-OCR-2 qwen2+ MODEL_TENSOR.V_RESMPL_QUERY_1024: "v.resample_query_1024", # Deepseek-OCR-2 qwen2+ # Granite4 Vision+ # qformer layers (bid => proj_id)+ # NOTE: Names align with A_QF_*+ MODEL_TENSOR.V_QF_SELF_ATTN_Q: "v.proj_blk.{bid}.self_attn_q",+ MODEL_TENSOR.V_QF_SELF_ATTN_K: "v.proj_blk.{bid}.self_attn_k",+ MODEL_TENSOR.V_QF_SELF_ATTN_V: "v.proj_blk.{bid}.self_attn_v",+ MODEL_TENSOR.V_QF_SELF_ATTN_O: "v.proj_blk.{bid}.self_attn_out",+ MODEL_TENSOR.V_QF_SELF_ATTN_NORM: "v.proj_blk.{bid}.self_attn_norm",+ MODEL_TENSOR.V_QF_CROSS_ATTN_Q: "v.proj_blk.{bid}.cross_attn_q",+ MODEL_TENSOR.V_QF_CROSS_ATTN_K: "v.proj_blk.{bid}.cross_attn_k",+ MODEL_TENSOR.V_QF_CROSS_ATTN_V: "v.proj_blk.{bid}.cross_attn_v",+ MODEL_TENSOR.V_QF_CROSS_ATTN_O: "v.proj_blk.{bid}.cross_attn_out",+ MODEL_TENSOR.V_QF_CROSS_ATTN_NORM: "v.proj_blk.{bid}.cross_attn_norm",+ MODEL_TENSOR.V_QF_FFN_UP: "v.proj_blk.{bid}.ffn_up",+ MODEL_TENSOR.V_QF_FFN_DOWN: "v.proj_blk.{bid}.ffn_down",+ MODEL_TENSOR.V_QF_FFN_NORM: "v.proj_blk.{bid}.ffn_norm",+ # multi-projector (bid => projector ID)+ MODEL_TENSOR.V_MULTI_PROJ_IMG_POS: "v.proj_blk.{bid}.img_pos",+ MODEL_TENSOR.V_MULTI_PROJ_QUERY: "v.proj_blk.{bid}.query",+ MODEL_TENSOR.V_MULTI_PROJ_NORM: "v.proj_blk.{bid}.norm",+ MODEL_TENSOR.V_MULTI_PROJ_LINEAR: "v.proj_blk.{bid}.linear",+ MODEL_TENSOR.V_MULTI_PROJ_POST_NORM: "v.proj_blk.{bid}.post_norm",++ # audio (mtmd)+ # note: all audio tensor names must use prefix "a." or "mm.a."+ MODEL_TENSOR.A_ENC_EMBD_POS: "a.position_embd",+ MODEL_TENSOR.A_ENC_EMBD_NORM: "a.position_embd_norm",+ MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS: "a.embd_to_logits",+ MODEL_TENSOR.A_ENC_INP_PROJ: "a.input_projection",+ MODEL_TENSOR.A_ENC_CONV1D: "a.conv1d.{bid}",+ MODEL_TENSOR.A_ENC_CONV2D: "a.conv2d.{bid}",+ MODEL_TENSOR.A_ENC_CONV_OUT: "a.conv_out",+ MODEL_TENSOR.A_ENC_CONV1D_NORM: "a.conv1d.{bid}.norm",+ MODEL_TENSOR.A_PRE_NORM: "a.pre_ln",+ MODEL_TENSOR.A_POST_NORM: "a.post_ln",+ MODEL_TENSOR.A_ENC_LAYER_PRE_NORM: "a.blk.{bid}.layer_pre_norm",+ MODEL_TENSOR.A_ENC_ATTN_Q: "a.blk.{bid}.attn_q",+ MODEL_TENSOR.A_ENC_ATTN_K: "a.blk.{bid}.attn_k",+ MODEL_TENSOR.A_ENC_ATTN_V: "a.blk.{bid}.attn_v",+ MODEL_TENSOR.A_ENC_ATTN_POST_NORM: "a.blk.{bid}.attn_post_norm",+ MODEL_TENSOR.A_ENC_ATTN_PRE_NORM: "a.blk.{bid}.attn_pre_norm",+ MODEL_TENSOR.A_ENC_ATTN_K_REL: "a.blk.{bid}.attn_k_rel",+ MODEL_TENSOR.A_ENC_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale",+ MODEL_TENSOR.A_ENC_INPUT_NORM: "a.blk.{bid}.ln1",+ MODEL_TENSOR.A_ENC_OUTPUT: "a.blk.{bid}.attn_out",+ MODEL_TENSOR.A_ENC_OUTPUT_NORM: "a.blk.{bid}.ln2",+ MODEL_TENSOR.A_ENC_FFN_NORM: "a.blk.{bid}.ffn_norm",+ MODEL_TENSOR.A_ENC_FFN_POST_NORM: "a.blk.{bid}.ffn_post_norm",+ MODEL_TENSOR.A_ENC_FFN_SCALE: "a.blk.{bid}.ffn_scale",+ MODEL_TENSOR.A_ENC_FFN_UP: "a.blk.{bid}.ffn_up",+ MODEL_TENSOR.A_ENC_FFN_GATE: "a.blk.{bid}.ffn_gate",+ MODEL_TENSOR.A_ENC_FFN_DOWN: "a.blk.{bid}.ffn_down",+ MODEL_TENSOR.A_ENC_FFN_NORM_1: "a.blk.{bid}.ffn_norm_1",+ MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: "a.blk.{bid}.ffn_post_norm_1",+ MODEL_TENSOR.A_ENC_FFN_SCALE_1: "a.blk.{bid}.ffn_scale_1",+ MODEL_TENSOR.A_ENC_FFN_UP_1: "a.blk.{bid}.ffn_up_1",+ MODEL_TENSOR.A_ENC_FFN_GATE_1: "a.blk.{bid}.ffn_gate_1",+ MODEL_TENSOR.A_ENC_FFN_DOWN_1: "a.blk.{bid}.ffn_down_1",+ MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: "a.downsample.conv",+ MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: "a.downsample.norm",+ MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: "a.rvq.codebook",+ MODEL_TENSOR.A_ENC_CONV_RES2: "a.blk.{bid}.res2.{xid}",+ MODEL_TENSOR.A_ENC_SE_CONV1: "a.blk.{bid}.se_conv1",+ MODEL_TENSOR.A_ENC_SE_CONV2: "a.blk.{bid}.se_conv2",+ MODEL_TENSOR.A_ENC_ASP_ATTN: "a.asp_attn",+ MODEL_TENSOR.A_ENC_ASP_TDNN: "a.asp_tdnn",+ MODEL_TENSOR.A_GEN_CODE_PROJ_IN: "a.gen.code.proj_in",+ MODEL_TENSOR.A_GEN_CODE_EMBD: "a.gen.code.embd",+ MODEL_TENSOR.A_GEN_CODE_HEAD: "a.gen.code.head",+ MODEL_TENSOR.A_GEN_CODE_OUT_EMBD: "a.gen.code.out_embd",+ MODEL_TENSOR.A_GEN_CODE_ATTN_NORM: "a.gen.code.blk.{bid}.ln1", # reuses the generic clip.cpp block loader (TN_LN_1)+ MODEL_TENSOR.A_GEN_CODE_ATTN_Q: "a.gen.code.blk.{bid}.attn_q",+ MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM: "a.gen.code.blk.{bid}.attn_q_norm",+ MODEL_TENSOR.A_GEN_CODE_ATTN_K: "a.gen.code.blk.{bid}.attn_k",+ MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM: "a.gen.code.blk.{bid}.attn_k_norm",+ MODEL_TENSOR.A_GEN_CODE_ATTN_V: "a.gen.code.blk.{bid}.attn_v",+ MODEL_TENSOR.A_GEN_CODE_ATTN_OUT: "a.gen.code.blk.{bid}.attn_out",+ MODEL_TENSOR.A_GEN_CODE_FFN_NORM: "a.gen.code.blk.{bid}.ln2", # reuses the generic clip.cpp block loader (TN_LN_2)+ MODEL_TENSOR.A_GEN_CODE_FFN_GATE: "a.gen.code.blk.{bid}.ffn_gate",+ MODEL_TENSOR.A_GEN_CODE_FFN_UP: "a.gen.code.blk.{bid}.ffn_up",+ MODEL_TENSOR.A_GEN_CODE_FFN_DOWN: "a.gen.code.blk.{bid}.ffn_down",+ MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM: "a.gen.code.output_norm",+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj",+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook",+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_IN: "a.gen.wav.quant.rest.in_proj",+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_OUT: "a.gen.wav.quant.rest.out_proj",+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_CB: "a.gen.wav.quant.rest.codebook",+ MODEL_TENSOR.A_GEN_WAV_PRE_CONV: "a.gen.wav.pre_conv",+ MODEL_TENSOR.A_GEN_WAV_TFM_IN_PROJ: "a.gen.wav.tfm.in_proj",+ MODEL_TENSOR.A_GEN_WAV_TFM_OUT_PROJ: "a.gen.wav.tfm.out_proj",+ MODEL_TENSOR.A_GEN_WAV_TFM_OUTPUT_NORM: "a.gen.wav.tfm.output_norm",+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM: "a.gen.wav.tfm.blk.{bid}.ln1",+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q: "a.gen.wav.tfm.blk.{bid}.attn_q",+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K: "a.gen.wav.tfm.blk.{bid}.attn_k",+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V: "a.gen.wav.tfm.blk.{bid}.attn_v",+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT: "a.gen.wav.tfm.blk.{bid}.attn_out",+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE: "a.gen.wav.tfm.blk.{bid}.ls1",+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM: "a.gen.wav.tfm.blk.{bid}.ln2",+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE: "a.gen.wav.tfm.blk.{bid}.ffn_gate",+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP: "a.gen.wav.tfm.blk.{bid}.ffn_up",+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN: "a.gen.wav.tfm.blk.{bid}.ffn_down",+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE: "a.gen.wav.tfm.blk.{bid}.ls2",+ MODEL_TENSOR.A_GEN_WAV_UP_CONV: "a.gen.wav.up.blk.{bid}.conv",+ MODEL_TENSOR.A_GEN_WAV_UP_DWCONV: "a.gen.wav.up.blk.{bid}.dwconv",+ MODEL_TENSOR.A_GEN_WAV_UP_NORM: "a.gen.wav.up.blk.{bid}.norm",+ MODEL_TENSOR.A_GEN_WAV_UP_PW1: "a.gen.wav.up.blk.{bid}.pw1",+ MODEL_TENSOR.A_GEN_WAV_UP_PW2: "a.gen.wav.up.blk.{bid}.pw2",+ MODEL_TENSOR.A_GEN_WAV_UP_GAMMA: "a.gen.wav.up.blk.{bid}.gamma",+ MODEL_TENSOR.A_GEN_WAV_DAC_ENTRY: "a.gen.wav.dac.entry",+ MODEL_TENSOR.A_GEN_WAV_DAC_UP_SNAKE: "a.gen.wav.dac.blk.{bid}.snake",+ MODEL_TENSOR.A_GEN_WAV_DAC_UP_CONV: "a.gen.wav.dac.blk.{bid}.conv",+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT1: "a.gen.wav.dac.blk.{bid}.res.{xid}.act1",+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV1: "a.gen.wav.dac.blk.{bid}.res.{xid}.conv1",+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT2: "a.gen.wav.dac.blk.{bid}.res.{xid}.act2",+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV2: "a.gen.wav.dac.blk.{bid}.res.{xid}.conv2",+ MODEL_TENSOR.A_GEN_WAV_DAC_POST_SNAKE: "a.gen.wav.dac.post_snake",+ MODEL_TENSOR.A_GEN_WAV_DAC_POST_CONV: "a.gen.wav.dac.post_conv",+ MODEL_TENSOR.A_ENC_SEANET_CONV_IN: "a.seanet.conv_in",+ MODEL_TENSOR.A_ENC_SEANET_CONV_OUT: "a.seanet.conv_out",+ MODEL_TENSOR.A_ENC_SEANET_RES_CONV1: "a.seanet.blk.{bid}.res_conv1",+ MODEL_TENSOR.A_ENC_SEANET_RES_CONV2: "a.seanet.blk.{bid}.res_conv2",+ MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV: "a.seanet.blk.{bid}.scale_conv",+ MODEL_TENSOR.A_ENC_ATTN_SCALE: "a.blk.{bid}.ls1",+ MODEL_TENSOR.A_ENC_FFN_SCALE_LS: "a.blk.{bid}.ls2",+ MODEL_TENSOR.A_ENC_SPEAKER_PROJ: "a.speaker_proj",+ MODEL_TENSOR.A_GEN_FLOW_INPUT_PROJ: "a.gen.flow.input_proj",+ MODEL_TENSOR.A_GEN_FLOW_COND_EMBD: "a.gen.flow.cond_embd",+ MODEL_TENSOR.A_GEN_FLOW_TIME_FREQS: "a.gen.flow.time.{bid}.freqs",+ MODEL_TENSOR.A_GEN_FLOW_TIME_UP: "a.gen.flow.time.{bid}.up",+ MODEL_TENSOR.A_GEN_FLOW_TIME_DOWN: "a.gen.flow.time.{bid}.down",+ MODEL_TENSOR.A_GEN_FLOW_TIME_NORM: "a.gen.flow.time.{bid}.norm",+ MODEL_TENSOR.A_GEN_FLOW_BLK_NORM: "a.gen.flow.blk.{bid}.norm",+ MODEL_TENSOR.A_GEN_FLOW_BLK_UP: "a.gen.flow.blk.{bid}.up",+ MODEL_TENSOR.A_GEN_FLOW_BLK_DOWN: "a.gen.flow.blk.{bid}.down",+ MODEL_TENSOR.A_GEN_FLOW_BLK_ADA: "a.gen.flow.blk.{bid}.ada",+ MODEL_TENSOR.A_GEN_FLOW_FINAL_ADA: "a.gen.flow.final.ada",+ MODEL_TENSOR.A_GEN_FLOW_FINAL_PROJ: "a.gen.flow.final.proj",+ MODEL_TENSOR.A_GEN_OUT_EOS: "a.gen.out_eos",+ MODEL_TENSOR.A_GEN_INPUT_LINEAR: "a.gen.input_linear",+ MODEL_TENSOR.A_GEN_EMB_MEAN: "a.gen.emb_mean",+ MODEL_TENSOR.A_GEN_EMB_STD: "a.gen.emb_std",+ MODEL_TENSOR.A_GEN_WAV_QUANT_OUT: "a.gen.wav.quant_out",+ MODEL_TENSOR.A_GEN_WAV_UPSAMPLE: "a.gen.wav.upsample",+ MODEL_TENSOR.A_GEN_WAV_SEANET_CONV_IN: "a.gen.wav.seanet.conv_in",+ MODEL_TENSOR.A_GEN_WAV_SEANET_CONV_OUT: "a.gen.wav.seanet.conv_out",+ MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV1: "a.gen.wav.seanet.blk.{bid}.res_conv1",+ MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV2: "a.gen.wav.seanet.blk.{bid}.res_conv2",+ MODEL_TENSOR.A_GEN_WAV_SEANET_SCALE_CONV: "a.gen.wav.seanet.blk.{bid}.scale_conv",+ MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",+ MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",+ MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",+ MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid",+ MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n+ MODEL_TENSOR.A_MM_SOFT_EMB_NORM: "mm.a.soft_emb_norm", # gemma3n+ MODEL_TENSOR.A_MM_EMBEDDING: "mm.a.embedding", # gemma3n+ MODEL_TENSOR.A_MM_HARD_EMB_NORM: "mm.a.hard_emb_norm", # gemma3n+ MODEL_TENSOR.A_MM_CODE_EMBD: "mm.a.code_embd",+ MODEL_TENSOR.A_MM_LOCAL_ATTN_Q: "mm.a.local_blk.{bid}.attn_q",+ MODEL_TENSOR.A_MM_LOCAL_ATTN_K: "mm.a.local_blk.{bid}.attn_k",+ MODEL_TENSOR.A_MM_LOCAL_ATTN_V: "mm.a.local_blk.{bid}.attn_v",+ MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT: "mm.a.local_blk.{bid}.attn_out",+ MODEL_TENSOR.A_MM_LOCAL_FFN_GATE: "mm.a.local_blk.{bid}.ffn_gate",+ MODEL_TENSOR.A_MM_LOCAL_FFN_UP: "mm.a.local_blk.{bid}.ffn_up",+ MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN: "mm.a.local_blk.{bid}.ffn_down",+ MODEL_TENSOR.A_MM_LOCAL_LN1: "mm.a.local_blk.{bid}.ln1",+ MODEL_TENSOR.A_MM_LOCAL_LN2: "mm.a.local_blk.{bid}.ln2",+ MODEL_TENSOR.A_MM_LOCAL_NORM: "mm.a.local_norm",+ MODEL_TENSOR.A_PER_DIM_K_SCALE: "a.blk.{bid}.per_dim_k_scale", # gemma4+ MODEL_TENSOR.A_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale", # gemma4+ # lfm2 audio+ MODEL_TENSOR.A_ENC_NORM_CONV: "a.blk.{bid}.norm_conv",+ MODEL_TENSOR.A_ENC_LINEAR_POS: "a.blk.{bid}.linear_pos",+ MODEL_TENSOR.A_ENC_POS_BIAS_U: "a.blk.{bid}.pos_bias_u",+ MODEL_TENSOR.A_ENC_POS_BIAS_V: "a.blk.{bid}.pos_bias_v",+ MODEL_TENSOR.A_ENC_OUT: "a.pre_encode.out",+ MODEL_TENSOR.A_ENC_CONV_DW: "a.blk.{bid}.conv_dw",+ MODEL_TENSOR.A_ENC_CONV_NORM: "a.blk.{bid}.conv_norm",+ MODEL_TENSOR.A_ENC_CONV_PW1: "a.blk.{bid}.conv_pw1",+ MODEL_TENSOR.A_ENC_CONV_PW2: "a.blk.{bid}.conv_pw2",+ MODEL_TENSOR.A_ENC_CONV_NORM_MEAN: "a.blk.{bid}.conv_norm_mean",+ MODEL_TENSOR.A_ENC_CONV_NORM_VAR: "a.blk.{bid}.conv_norm_var",+ MODEL_TENSOR.A_ENC_MEL_FILTERS: "a.mel_filters",+ MODEL_TENSOR.A_ENC_WINDOW: "a.window",+ MODEL_TENSOR.A_CTC_OUT: "a.enc_ctc_out",+ MODEL_TENSOR.A_CTC_OUT_MID: "a.enc_ctc_out_mid",+ MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB: "a.blk.{bid}.attn_rel_pos_emb",+ # qformer projector+ MODEL_TENSOR.A_QF_PROJ_QUERY: "a.proj_query",+ MODEL_TENSOR.A_QF_PROJ_NORM: "a.proj_norm",+ MODEL_TENSOR.A_QF_PROJ_LINEAR: "a.proj_linear",+ MODEL_TENSOR.A_QF_SELF_ATTN_Q: "a.proj_blk.{bid}.self_attn_q",+ MODEL_TENSOR.A_QF_SELF_ATTN_K: "a.proj_blk.{bid}.self_attn_k",+ MODEL_TENSOR.A_QF_SELF_ATTN_V: "a.proj_blk.{bid}.self_attn_v",+ MODEL_TENSOR.A_QF_SELF_ATTN_O: "a.proj_blk.{bid}.self_attn_out",+ MODEL_TENSOR.A_QF_SELF_ATTN_NORM: "a.proj_blk.{bid}.self_attn_norm",+ MODEL_TENSOR.A_QF_CROSS_ATTN_Q: "a.proj_blk.{bid}.cross_attn_q",+ MODEL_TENSOR.A_QF_CROSS_ATTN_K: "a.proj_blk.{bid}.cross_attn_k",+ MODEL_TENSOR.A_QF_CROSS_ATTN_V: "a.proj_blk.{bid}.cross_attn_v",+ MODEL_TENSOR.A_QF_CROSS_ATTN_O: "a.proj_blk.{bid}.cross_attn_out",+ MODEL_TENSOR.A_QF_CROSS_ATTN_NORM: "a.proj_blk.{bid}.cross_attn_norm",+ MODEL_TENSOR.A_QF_FFN_UP: "a.proj_blk.{bid}.ffn_up",+ MODEL_TENSOR.A_QF_FFN_DOWN: "a.proj_blk.{bid}.ffn_down",+ MODEL_TENSOR.A_QF_FFN_NORM: "a.proj_blk.{bid}.ffn_norm",+ # NextN/MTP+ MODEL_TENSOR.NEXTN_PROJ_PRE: "nextn.pre_projection",+ MODEL_TENSOR.NEXTN_PROJ_POST: "nextn.post_projection",+ MODEL_TENSOR.NEXTN_EH_PROJ: "blk.{bid}.nextn.eh_proj",+ MODEL_TENSOR.NEXTN_EMBED_TOKENS: "blk.{bid}.nextn.embed_tokens",+ MODEL_TENSOR.NEXTN_ENORM: "blk.{bid}.nextn.enorm",+ MODEL_TENSOR.NEXTN_HNORM: "blk.{bid}.nextn.hnorm",+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD: "blk.{bid}.nextn.shared_head_head",+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: "blk.{bid}.nextn.shared_head_norm",+ MODEL_TENSOR.NEXTN_HC_HEAD_NORM: "blk.{bid}.nextn.hc_head_norm",+ MODEL_TENSOR.NEXTN_HC_HEAD_DOWN: "blk.{bid}.nextn.hc_head_down",+ MODEL_TENSOR.NEXTN_HC_HEAD_UP: "blk.{bid}.nextn.hc_head_up",+ MODEL_TENSOR.FC: "fc",+ MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1",+ MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2",+ MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj",+ MODEL_TENSOR.DFLASH_ATTN_CONV_BASE: "blk.{bid}.attn_conv_base",+ MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ: "blk.{bid}.attn_conv_proj",+ MODEL_TENSOR.DFLASH_FFN_CONV_BASE: "blk.{bid}.ffn_conv_base",+ MODEL_TENSOR.DFLASH_FFN_CONV_PROJ: "blk.{bid}.ffn_conv_proj",+ MODEL_TENSOR.DFLASH_SELECTOR_PREV: "selector_predecessor",+ MODEL_TENSOR.DFLASH_SELECTOR_NEXT: "selector_successor",+ MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN: "selector_hidden",+ MODEL_TENSOR.D2T: "d2t",+ MODEL_TENSOR.ATTN_V_GATE: "blk.{bid}.attn_v_gate",+ MODEL_TENSOR.ATTN_V_EXP: "blk.{bid}.attn_v_exps",+}++MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {+ MODEL_ARCH.MMPROJ: [+ MODEL_TENSOR.V_MMPROJ,+ MODEL_TENSOR.V_MMPROJ_FC,+ MODEL_TENSOR.V_MMPROJ_MLP,+ MODEL_TENSOR.V_MMPROJ_PEG,+ MODEL_TENSOR.V_ENC_EMBD_CLS,+ MODEL_TENSOR.V_ENC_EMBD_PATCH,+ MODEL_TENSOR.V_ENC_EMBD_NORM,+ MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM,+ MODEL_TENSOR.V_ENC_EMBD_POS,+ MODEL_TENSOR.V_ENC_EMBD_IMGNL,+ MODEL_TENSOR.V_ENC_EMBD_VSEP,+ MODEL_TENSOR.V_TOK_EMBD_SEP,+ MODEL_TENSOR.V_ENC_INPUT_NORM,+ MODEL_TENSOR.V_ENC_ATTN_QKV,+ MODEL_TENSOR.V_ENC_ATTN_Q,+ MODEL_TENSOR.V_ENC_ATTN_Q_NORM,+ MODEL_TENSOR.V_ENC_ATTN_K,+ MODEL_TENSOR.V_ENC_ATTN_K_NORM,+ MODEL_TENSOR.V_ENC_ATTN_V,+ MODEL_TENSOR.V_ENC_ATTN_O,+ MODEL_TENSOR.V_ENC_ATTN_O_NORM,+ MODEL_TENSOR.V_ENC_ATTN_SINKS,+ MODEL_TENSOR.V_ENC_POST_ATTN_NORM,+ MODEL_TENSOR.V_ENC_FFN_UP,+ MODEL_TENSOR.V_ENC_FFN_GATE,+ MODEL_TENSOR.V_ENC_FFN_DOWN,+ MODEL_TENSOR.V_ENC_FFN_GATE_INP,+ MODEL_TENSOR.V_ENC_FFN_GATE_EXPS,+ MODEL_TENSOR.V_ENC_FFN_UP_EXPS,+ MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS,+ MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B,+ MODEL_TENSOR.V_ENC_ATTN_POST_NORM,+ MODEL_TENSOR.V_ENC_FFN_POST_NORM,+ MODEL_TENSOR.V_LAYER_SCALE_1,+ MODEL_TENSOR.V_LAYER_SCALE_2,+ MODEL_TENSOR.V_LAYER_OUT_SCALE,+ MODEL_TENSOR.V_PRE_NORM,+ MODEL_TENSOR.V_POST_NORM,+ MODEL_TENSOR.V_MM_POST_NORM,+ MODEL_TENSOR.V_MM_INP_PROJ,+ MODEL_TENSOR.V_MM_INP_NORM,+ MODEL_TENSOR.V_MM_SOFT_EMB_NORM,+ MODEL_TENSOR.V_MM_EMBEDDING,+ MODEL_TENSOR.V_MM_HARD_EMB_NORM,+ MODEL_TENSOR.V_ENC_CONV_STEM,+ MODEL_TENSOR.V_ENC_CONV_STEM_NORM,+ MODEL_TENSOR.V_ENC_MSFA_EXP,+ MODEL_TENSOR.V_ENC_MSFA_EXP_NORM,+ MODEL_TENSOR.V_ENC_MSFA_PROJ,+ MODEL_TENSOR.V_ENC_MSFA_PROJ_NORM,+ MODEL_TENSOR.V_ENC_MSFA_NORM,+ MODEL_TENSOR.V_RESMPL_POS_EMBD_K,+ MODEL_TENSOR.V_RESMPL_ATTN_Q,+ MODEL_TENSOR.V_RESMPL_ATTN_K,+ MODEL_TENSOR.V_RESMPL_ATTN_V,+ MODEL_TENSOR.V_RESMPL_ATTN_OUT,+ MODEL_TENSOR.V_RESMPL_KV,+ MODEL_TENSOR.V_RESMPL_KV_NORM,+ MODEL_TENSOR.V_RESMPL_POST_NORM,+ MODEL_TENSOR.V_RESMPL_Q_NORM,+ MODEL_TENSOR.V_RESMPL_PROJ,+ MODEL_TENSOR.V_RESMPL_QUERY,+ MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK,+ MODEL_TENSOR.V_TOK_EMBD_IMG_START,+ MODEL_TENSOR.V_TOK_EMBD_IMG_END,+ MODEL_TENSOR.V_TOK_EMBD_IMG_PAD,+ MODEL_TENSOR.V_MM_PATCH_MERGER,+ MODEL_TENSOR.V_MM_MERGER_FC1,+ MODEL_TENSOR.V_MM_MERGER_FC2,+ MODEL_TENSOR.V_DS_NORM,+ MODEL_TENSOR.V_DS_FC1,+ MODEL_TENSOR.V_DS_FC2,+ MODEL_TENSOR.V_MERGER_LN1,+ MODEL_TENSOR.V_MERGER_ATTN_Q,+ MODEL_TENSOR.V_MERGER_ATTN_K,+ MODEL_TENSOR.V_MERGER_ATTN_V,+ MODEL_TENSOR.V_MERGER_ATTN_O,+ MODEL_TENSOR.V_MERGER_DS_LN,+ MODEL_TENSOR.V_MERGER_DS_UP,+ MODEL_TENSOR.V_MERGER_DS_DOWN,+ MODEL_TENSOR.V_MM_POST_FC_NORM,+ MODEL_TENSOR.V_MM_UP,+ MODEL_TENSOR.V_MM_DOWN,+ MODEL_TENSOR.V_MM_GATE,+ MODEL_TENSOR.V_TOK_BOI,+ MODEL_TENSOR.V_TOK_EOI,+ MODEL_TENSOR.V_MM_PRE_NORM,+ MODEL_TENSOR.V_TOK_IMG_BEGIN,+ MODEL_TENSOR.V_TOK_IMG_END,+ MODEL_TENSOR.V_STD_BIAS,+ MODEL_TENSOR.V_STD_SCALE,+ MODEL_TENSOR.V_SAM_POS_EMBD,+ MODEL_TENSOR.V_SAM_PATCH_EMBD,+ MODEL_TENSOR.V_SAM_PRE_NORM,+ MODEL_TENSOR.V_SAM_POST_NORM,+ MODEL_TENSOR.V_SAM_ATTN_POS_H,+ MODEL_TENSOR.V_SAM_ATTN_POS_W,+ MODEL_TENSOR.V_SAM_ATTN_QKV,+ MODEL_TENSOR.V_SAM_ATTN_OUT,+ MODEL_TENSOR.V_SAM_MLP_LIN_1,+ MODEL_TENSOR.V_SAM_MLP_LIN_2,+ MODEL_TENSOR.V_SAM_NECK,+ MODEL_TENSOR.V_SAM_NET_2,+ MODEL_TENSOR.V_SAM_NET_3,+ MODEL_TENSOR.V_RESMPL_QUERY_768,+ MODEL_TENSOR.V_RESMPL_QUERY_1024,+ MODEL_TENSOR.V_PROJ_NORM,+ MODEL_TENSOR.V_QF_PROJ_QUERY,+ MODEL_TENSOR.V_QF_PROJ_NORM,+ MODEL_TENSOR.V_QF_PROJ_LINEAR,+ MODEL_TENSOR.V_QF_SELF_ATTN_Q,+ MODEL_TENSOR.V_QF_SELF_ATTN_K,+ MODEL_TENSOR.V_QF_SELF_ATTN_V,+ MODEL_TENSOR.V_QF_SELF_ATTN_O,+ MODEL_TENSOR.V_QF_SELF_ATTN_NORM,+ MODEL_TENSOR.V_QF_CROSS_ATTN_Q,+ MODEL_TENSOR.V_QF_CROSS_ATTN_K,+ MODEL_TENSOR.V_QF_CROSS_ATTN_V,+ MODEL_TENSOR.V_QF_CROSS_ATTN_O,+ MODEL_TENSOR.V_QF_CROSS_ATTN_NORM,+ MODEL_TENSOR.V_QF_FFN_UP,+ MODEL_TENSOR.V_QF_FFN_DOWN,+ MODEL_TENSOR.V_QF_FFN_NORM,+ MODEL_TENSOR.V_QF_PROJ_NORM,+ MODEL_TENSOR.V_MULTI_PROJ_IMG_POS,+ MODEL_TENSOR.V_MULTI_PROJ_QUERY,+ MODEL_TENSOR.V_MULTI_PROJ_LINEAR,+ MODEL_TENSOR.V_MULTI_PROJ_NORM,+ MODEL_TENSOR.V_MULTI_PROJ_POST_NORM,+ # audio+ MODEL_TENSOR.A_ENC_EMBD_POS,+ MODEL_TENSOR.A_ENC_EMBD_NORM,+ MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS,+ MODEL_TENSOR.A_ENC_INP_PROJ,+ MODEL_TENSOR.A_ENC_CONV1D,+ MODEL_TENSOR.A_ENC_CONV2D,+ MODEL_TENSOR.A_ENC_CONV_OUT,+ MODEL_TENSOR.A_ENC_CONV1D_NORM,+ MODEL_TENSOR.A_PRE_NORM,+ MODEL_TENSOR.A_POST_NORM,+ MODEL_TENSOR.A_ENC_LAYER_PRE_NORM,+ MODEL_TENSOR.A_ENC_ATTN_Q,+ MODEL_TENSOR.A_ENC_ATTN_K,+ MODEL_TENSOR.A_ENC_ATTN_V,+ MODEL_TENSOR.A_ENC_ATTN_POST_NORM,+ MODEL_TENSOR.A_ENC_ATTN_PRE_NORM,+ MODEL_TENSOR.A_ENC_ATTN_K_REL,+ MODEL_TENSOR.A_ENC_PER_DIM_SCALE,+ MODEL_TENSOR.A_ENC_INPUT_NORM,+ MODEL_TENSOR.A_ENC_OUTPUT,+ MODEL_TENSOR.A_ENC_OUTPUT_NORM,+ MODEL_TENSOR.A_ENC_FFN_NORM,+ MODEL_TENSOR.A_ENC_FFN_POST_NORM,+ MODEL_TENSOR.A_ENC_FFN_SCALE,+ MODEL_TENSOR.A_ENC_FFN_UP,+ MODEL_TENSOR.A_ENC_FFN_GATE,+ MODEL_TENSOR.A_ENC_FFN_DOWN,+ MODEL_TENSOR.A_ENC_FFN_NORM_1,+ MODEL_TENSOR.A_ENC_FFN_POST_NORM_1,+ MODEL_TENSOR.A_ENC_FFN_SCALE_1,+ MODEL_TENSOR.A_ENC_FFN_UP_1,+ MODEL_TENSOR.A_ENC_FFN_GATE_1,+ MODEL_TENSOR.A_ENC_FFN_DOWN_1,+ MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV,+ MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM,+ MODEL_TENSOR.A_ENC_RVQ_CODEBOOK,+ MODEL_TENSOR.A_MMPROJ,+ MODEL_TENSOR.A_MMPROJ_FC,+ MODEL_TENSOR.A_MM_NORM_PRE,+ MODEL_TENSOR.A_MM_NORM_MID,+ MODEL_TENSOR.A_MM_CODE_EMBD,+ MODEL_TENSOR.A_MM_LOCAL_ATTN_Q,+ MODEL_TENSOR.A_MM_LOCAL_ATTN_K,+ MODEL_TENSOR.A_MM_LOCAL_ATTN_V,+ MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT,+ MODEL_TENSOR.A_MM_LOCAL_FFN_GATE,+ MODEL_TENSOR.A_MM_LOCAL_FFN_UP,+ MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN,+ MODEL_TENSOR.A_MM_LOCAL_LN1,+ MODEL_TENSOR.A_MM_LOCAL_LN2,+ MODEL_TENSOR.A_MM_LOCAL_NORM,+ MODEL_TENSOR.A_ENC_NORM_CONV,+ MODEL_TENSOR.A_ENC_LINEAR_POS,+ MODEL_TENSOR.A_ENC_POS_BIAS_U,+ MODEL_TENSOR.A_ENC_POS_BIAS_V,+ MODEL_TENSOR.A_ENC_OUT,+ MODEL_TENSOR.A_ENC_CONV_DW,+ MODEL_TENSOR.A_ENC_CONV_NORM,+ MODEL_TENSOR.A_ENC_CONV_PW1,+ MODEL_TENSOR.A_ENC_CONV_PW2,+ MODEL_TENSOR.A_ENC_CONV_RES2,+ MODEL_TENSOR.A_ENC_SE_CONV1,+ MODEL_TENSOR.A_ENC_SE_CONV2,+ MODEL_TENSOR.A_ENC_ASP_ATTN,+ MODEL_TENSOR.A_ENC_ASP_TDNN,+ MODEL_TENSOR.A_GEN_CODE_PROJ_IN,+ MODEL_TENSOR.A_GEN_CODE_EMBD,+ MODEL_TENSOR.A_GEN_CODE_HEAD,+ MODEL_TENSOR.A_GEN_CODE_OUT_EMBD,+ MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,+ MODEL_TENSOR.A_GEN_CODE_ATTN_Q,+ MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM,+ MODEL_TENSOR.A_GEN_CODE_ATTN_K,+ MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM,+ MODEL_TENSOR.A_GEN_CODE_ATTN_V,+ MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,+ MODEL_TENSOR.A_GEN_CODE_FFN_NORM,+ MODEL_TENSOR.A_GEN_CODE_FFN_GATE,+ MODEL_TENSOR.A_GEN_CODE_FFN_UP,+ MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,+ MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM,+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,+ MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_IN,+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_OUT,+ MODEL_TENSOR.A_GEN_WAV_QUANT_REST_CB,+ MODEL_TENSOR.A_GEN_WAV_PRE_CONV,+ MODEL_TENSOR.A_GEN_WAV_TFM_IN_PROJ,+ MODEL_TENSOR.A_GEN_WAV_TFM_OUT_PROJ,+ MODEL_TENSOR.A_GEN_WAV_TFM_OUTPUT_NORM,+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM,+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q,+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K,+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V,+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT,+ MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE,+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM,+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE,+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,+ MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE,+ MODEL_TENSOR.A_GEN_WAV_UP_CONV,+ MODEL_TENSOR.A_GEN_WAV_UP_DWCONV,+ MODEL_TENSOR.A_GEN_WAV_UP_NORM,+ MODEL_TENSOR.A_GEN_WAV_UP_PW1,+ MODEL_TENSOR.A_GEN_WAV_UP_PW2,+ MODEL_TENSOR.A_GEN_WAV_UP_GAMMA,+ MODEL_TENSOR.A_GEN_WAV_DAC_ENTRY,+ MODEL_TENSOR.A_GEN_WAV_DAC_UP_SNAKE,+ MODEL_TENSOR.A_GEN_WAV_DAC_UP_CONV,+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT1,+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV1,+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_ACT2,+ MODEL_TENSOR.A_GEN_WAV_DAC_RES_CONV2,+ MODEL_TENSOR.A_GEN_WAV_DAC_POST_SNAKE,+ MODEL_TENSOR.A_GEN_WAV_DAC_POST_CONV,+ MODEL_TENSOR.A_ENC_SEANET_CONV_IN,+ MODEL_TENSOR.A_ENC_SEANET_CONV_OUT,+ MODEL_TENSOR.A_ENC_SEANET_RES_CONV1,+ MODEL_TENSOR.A_ENC_SEANET_RES_CONV2,+ MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV,+ MODEL_TENSOR.A_ENC_ATTN_SCALE,+ MODEL_TENSOR.A_ENC_FFN_SCALE_LS,+ MODEL_TENSOR.A_ENC_SPEAKER_PROJ,+ MODEL_TENSOR.A_GEN_FLOW_INPUT_PROJ,+ MODEL_TENSOR.A_GEN_FLOW_COND_EMBD,+ MODEL_TENSOR.A_GEN_FLOW_TIME_FREQS,+ MODEL_TENSOR.A_GEN_FLOW_TIME_UP,+ MODEL_TENSOR.A_GEN_FLOW_TIME_DOWN,+ MODEL_TENSOR.A_GEN_FLOW_TIME_NORM,+ MODEL_TENSOR.A_GEN_FLOW_BLK_NORM,+ MODEL_TENSOR.A_GEN_FLOW_BLK_UP,+ MODEL_TENSOR.A_GEN_FLOW_BLK_DOWN,+ MODEL_TENSOR.A_GEN_FLOW_BLK_ADA,+ MODEL_TENSOR.A_GEN_FLOW_FINAL_ADA,+ MODEL_TENSOR.A_GEN_FLOW_FINAL_PROJ,+ MODEL_TENSOR.A_GEN_OUT_EOS,+ MODEL_TENSOR.A_GEN_INPUT_LINEAR,+ MODEL_TENSOR.A_GEN_EMB_MEAN,+ MODEL_TENSOR.A_GEN_EMB_STD,+ MODEL_TENSOR.A_GEN_WAV_QUANT_OUT,+ MODEL_TENSOR.A_GEN_WAV_UPSAMPLE,+ MODEL_TENSOR.A_GEN_WAV_SEANET_CONV_IN,+ MODEL_TENSOR.A_GEN_WAV_SEANET_CONV_OUT,+ MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV1,+ MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV2,+ MODEL_TENSOR.A_GEN_WAV_SEANET_SCALE_CONV,+ MODEL_TENSOR.A_ENC_CONV_NORM_MEAN,+ MODEL_TENSOR.A_ENC_CONV_NORM_VAR,+ MODEL_TENSOR.A_ENC_MEL_FILTERS,+ MODEL_TENSOR.A_ENC_WINDOW,+ MODEL_TENSOR.A_MM_INP_PROJ,+ MODEL_TENSOR.A_MM_SOFT_EMB_NORM,+ MODEL_TENSOR.A_MM_EMBEDDING,+ MODEL_TENSOR.A_MM_HARD_EMB_NORM,+ MODEL_TENSOR.A_PER_DIM_K_SCALE,+ MODEL_TENSOR.A_PER_DIM_SCALE,+ MODEL_TENSOR.A_CTC_OUT,+ MODEL_TENSOR.A_CTC_OUT_MID,+ MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB,+ # qformer projector+ MODEL_TENSOR.A_QF_PROJ_QUERY,+ MODEL_TENSOR.A_QF_PROJ_NORM,+ MODEL_TENSOR.A_QF_PROJ_LINEAR,+ MODEL_TENSOR.A_QF_SELF_ATTN_Q,+ MODEL_TENSOR.A_QF_SELF_ATTN_K,+ MODEL_TENSOR.A_QF_SELF_ATTN_V,+ MODEL_TENSOR.A_QF_SELF_ATTN_O,+ MODEL_TENSOR.A_QF_SELF_ATTN_NORM,+ MODEL_TENSOR.A_QF_CROSS_ATTN_Q,+ MODEL_TENSOR.A_QF_CROSS_ATTN_K,+ MODEL_TENSOR.A_QF_CROSS_ATTN_V,+ MODEL_TENSOR.A_QF_CROSS_ATTN_O,+ MODEL_TENSOR.A_QF_CROSS_ATTN_NORM,+ MODEL_TENSOR.A_QF_FFN_UP,+ MODEL_TENSOR.A_QF_FFN_DOWN,+ MODEL_TENSOR.A_QF_FFN_NORM,+ ],+ MODEL_ARCH.LLAMA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.LLAMA4: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ ],+ MODEL_ARCH.DECI: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.GROK: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.ATTN_OUT_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_POST_NORM,+ MODEL_TENSOR.LAYER_OUT_NORM,+ ],+ MODEL_ARCH.GPTNEOX: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.FALCON: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_NORM_2,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.BAICHUAN: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.STARCODER: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.POS_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.BERT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.TOKEN_TYPES,+ MODEL_TENSOR.POS_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_OUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.LAYER_OUT_NORM,+ MODEL_TENSOR.CLS,+ MODEL_TENSOR.CLS_OUT,+ ],+ MODEL_ARCH.MODERN_BERT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.TOKEN_TYPES,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.CLS,+ MODEL_TENSOR.CLS_OUT,+ MODEL_TENSOR.CLS_NORM,+ ],+ MODEL_ARCH.NOMIC_BERT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.TOKEN_TYPES,+ MODEL_TENSOR.POS_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_OUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.LAYER_OUT_NORM,+ ],+ MODEL_ARCH.NOMIC_BERT_MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.TOKEN_TYPES,+ MODEL_TENSOR.POS_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_OUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.LAYER_OUT_NORM,+ ],+ MODEL_ARCH.NEO_BERT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ENC_OUTPUT_NORM,+ MODEL_TENSOR.CLS,+ MODEL_TENSOR.CLS_OUT,+ ],+ MODEL_ARCH.JINA_BERT_V2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.TOKEN_TYPES,+ MODEL_TENSOR.ATTN_NORM_2,+ MODEL_TENSOR.ATTN_OUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.LAYER_OUT_NORM,+ MODEL_TENSOR.CLS,+ ],+ MODEL_ARCH.JINA_BERT_V3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.TOKEN_TYPES,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_OUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.LAYER_OUT_NORM,+ ],+ MODEL_ARCH.EUROBERT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_DOWN,+ ],+ MODEL_ARCH.MPT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_ACT,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.POS_EMBD,+ ],+ MODEL_ARCH.GPTJ: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.REFACT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.BLOOM: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.STABLELM: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ ],+ MODEL_ARCH.QWEN: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.QWEN2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.DREAM: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.LLADA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.QWEN2VL: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.QWEN2MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_INP_SHEXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ ],+ MODEL_ARCH.QWEN3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.QWEN3MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.QWEN3NEXT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_INP_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_IN,+ MODEL_TENSOR.SSM_BETA_ALPHA,+ MODEL_TENSOR.SSM_OUT,+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.HRM_TEXT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.HRM_Z_L_INIT,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.QWEN3VL: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.QWEN3VLMOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.QWEN35: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.CLS_OUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_BETA,+ MODEL_TENSOR.SSM_ALPHA,+ MODEL_TENSOR.SSM_OUT,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.CLEF: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_BETA,+ MODEL_TENSOR.SSM_ALPHA,+ MODEL_TENSOR.SSM_OUT,+ # decision head+ MODEL_TENSOR.TOKEN_TYPES,+ MODEL_TENSOR.DEC_ATTN_NORM,+ MODEL_TENSOR.DEC_ATTN_Q,+ MODEL_TENSOR.DEC_ATTN_K,+ MODEL_TENSOR.DEC_ATTN_V,+ MODEL_TENSOR.DEC_ATTN_OUT,+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM,+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV,+ MODEL_TENSOR.DEC_CROSS_ATTN_Q,+ MODEL_TENSOR.DEC_CROSS_ATTN_K,+ MODEL_TENSOR.DEC_CROSS_ATTN_V,+ MODEL_TENSOR.DEC_CROSS_ATTN_OUT,+ MODEL_TENSOR.DEC_FFN_NORM,+ MODEL_TENSOR.DEC_FFN_DOWN,+ MODEL_TENSOR.DEC_FFN_UP,+ MODEL_TENSOR.DECISION_HIDDEN_NORM,+ MODEL_TENSOR.DECISION_PROJ_MEMORY,+ MODEL_TENSOR.DECISION_PROJ_QUESTION,+ MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION,+ MODEL_TENSOR.DECISION_PROJ_GLOBAL,+ MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT,+ MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL,+ MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM,+ MODEL_TENSOR.DECISION_FIELD_NORM,+ MODEL_TENSOR.DECISION_OPTION_NORM,+ MODEL_TENSOR.DECISION_SCALES,+ MODEL_TENSOR.DECISION_SCORER,+ MODEL_TENSOR.DECISION_SCORER_OUT,+ ],+ MODEL_ARCH.VLLM_DECISION2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ # decision head+ MODEL_TENSOR.DECISION_OPTION_NORM,+ MODEL_TENSOR.DECISION_QUERY_NORM,+ MODEL_TENSOR.DECISION_KEY,+ MODEL_TENSOR.DECISION_QUERY,+ MODEL_TENSOR.DECISION_OPTION_MLP,+ MODEL_TENSOR.DECISION_QUERY_MLP,+ MODEL_TENSOR.DECISION_SCORER_OUT,+ ],+ MODEL_ARCH.QWEN35MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_INP_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_BETA,+ MODEL_TENSOR.SSM_ALPHA,+ MODEL_TENSOR.SSM_OUT,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.QWEN4EXP: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ # no OUTPUT_NORM / ATTN_NORM / ATTN_POST_NORM: hyper-connections replace every layer norm+ MODEL_TENSOR.HC_HEAD_NORM,+ MODEL_TENSOR.HC_HEAD_DOWN,+ MODEL_TENSOR.HC_HEAD_UP,+ MODEL_TENSOR.HC_ATTN_NORM,+ MODEL_TENSOR.HC_ATTN_DOWN,+ MODEL_TENSOR.HC_ATTN_UP,+ MODEL_TENSOR.HC_ATTN_INJECT,+ MODEL_TENSOR.HC_FFN_NORM,+ MODEL_TENSOR.HC_FFN_DOWN,+ MODEL_TENSOR.HC_FFN_UP,+ MODEL_TENSOR.HC_FFN_INJECT,+ # full attention layers: ATTN_Q holds [q|gate] interleaved per head+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.INDEXER_Q_PROJ,+ MODEL_TENSOR.INDEXER_K_PROJ,+ MODEL_TENSOR.INDEXER_Q_NORM,+ MODEL_TENSOR.INDEXER_K_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_BETA,+ MODEL_TENSOR.SSM_ALPHA,+ MODEL_TENSOR.SSM_OUT,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_INP_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,+ MODEL_TENSOR.PLE_KEY,+ MODEL_TENSOR.PLE_VALUE,+ MODEL_TENSOR.PLE_NORM_KEY,+ MODEL_TENSOR.PLE_NORM_QUERY,+ MODEL_TENSOR.PLE_NORM_CONV,+ MODEL_TENSOR.PLE_CONV1D,+ # MTP block: [fc_embedding | fc_hidden] as eh_proj, its own hyper-connection mixer as the head+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_HC_HEAD_NORM,+ MODEL_TENSOR.NEXTN_HC_HEAD_DOWN,+ MODEL_TENSOR.NEXTN_HC_HEAD_UP,+ ],+ MODEL_ARCH.PLAMO: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.PLAMO2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_POST_NORM,+ MODEL_TENSOR.SSM_IN,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_X,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_D,+ MODEL_TENSOR.SSM_OUT,+ MODEL_TENSOR.SSM_DT_NORM,+ MODEL_TENSOR.SSM_B_NORM,+ MODEL_TENSOR.SSM_C_NORM,+ ],+ MODEL_ARCH.PLAMO3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_POST_NORM,+ ],+ MODEL_ARCH.GPT2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.POS_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.PHI2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.PHI3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FACTORS_LONG,+ MODEL_TENSOR.ROPE_FACTORS_SHORT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.PHIMOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FACTORS_LONG,+ MODEL_TENSOR.ROPE_FACTORS_SHORT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.CODESHELL: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.POS_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.ORION: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.INTERNLM2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.MINICPM: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ROPE_FACTORS_LONG,+ MODEL_TENSOR.ROPE_FACTORS_SHORT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.MINICPM3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FACTORS_LONG,+ MODEL_TENSOR.ROPE_FACTORS_SHORT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.GEMMA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_NORM,+ ],+ MODEL_ARCH.GEMMA2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ ],+ MODEL_ARCH.GEMMA3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ ],+ MODEL_ARCH.GEMMA3N: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ # altup / laurel+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,+ MODEL_TENSOR.PER_LAYER_MODEL_PROJ,+ MODEL_TENSOR.PER_LAYER_INP_GATE,+ MODEL_TENSOR.PER_LAYER_PROJ,+ MODEL_TENSOR.PER_LAYER_PROJ_NORM,+ MODEL_TENSOR.PER_LAYER_POST_NORM,+ MODEL_TENSOR.ALTUP_PROJ,+ MODEL_TENSOR.ALTUP_UNEMBD_PROJ,+ MODEL_TENSOR.ALTUP_CORRECT_COEF,+ MODEL_TENSOR.ALTUP_CORRECT_SCALE,+ MODEL_TENSOR.ALTUP_PREDICT_COEF,+ MODEL_TENSOR.ALTUP_ROUTER,+ MODEL_TENSOR.ALTUP_ROUTER_NORM,+ MODEL_TENSOR.LAUREL_L,+ MODEL_TENSOR.LAUREL_R,+ MODEL_TENSOR.LAUREL_POST_NORM,+ ],+ MODEL_ARCH.GEMMA4: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_PRE_NORM_2,+ MODEL_TENSOR.FFN_POST_NORM,+ MODEL_TENSOR.FFN_POST_NORM_1,+ MODEL_TENSOR.FFN_POST_NORM_2,+ MODEL_TENSOR.LAYER_OUT_SCALE,+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,+ MODEL_TENSOR.PER_LAYER_MODEL_PROJ,+ MODEL_TENSOR.PER_LAYER_INP_GATE,+ MODEL_TENSOR.PER_LAYER_PROJ,+ MODEL_TENSOR.PER_LAYER_PROJ_NORM,+ MODEL_TENSOR.PER_LAYER_POST_NORM,+ ],+ MODEL_ARCH.GEMMA4_ASSISTANT: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.MASKED_EMBD_CENTROIDS,+ MODEL_TENSOR.MASKED_EMBD_ORDERING,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.NEXTN_PROJ_PRE,+ MODEL_TENSOR.NEXTN_PROJ_POST,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ MODEL_TENSOR.LAYER_OUT_SCALE,+ ],+ MODEL_ARCH.GEMMA_EMBEDDING: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.DENSE_2_OUT,+ MODEL_TENSOR.DENSE_3_OUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ ],+ MODEL_ARCH.GEMMA_EMBEDDING2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ MODEL_TENSOR.LAYER_OUT_SCALE,+ MODEL_TENSOR.PER_LAYER_MODEL_PROJ,+ MODEL_TENSOR.PER_LAYER_INP_GATE,+ MODEL_TENSOR.PER_LAYER_PROJ,+ MODEL_TENSOR.PER_LAYER_PROJ_NORM,+ MODEL_TENSOR.PER_LAYER_POST_NORM,+ ],+ MODEL_ARCH.STARCODER2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.RWKV6: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_NORM_2,+ MODEL_TENSOR.TIME_MIX_W1,+ MODEL_TENSOR.TIME_MIX_W2,+ MODEL_TENSOR.TIME_MIX_LERP_X,+ MODEL_TENSOR.TIME_MIX_LERP_K,+ MODEL_TENSOR.TIME_MIX_LERP_V,+ MODEL_TENSOR.TIME_MIX_LERP_R,+ MODEL_TENSOR.TIME_MIX_LERP_G,+ MODEL_TENSOR.TIME_MIX_LERP_W,+ MODEL_TENSOR.TIME_MIX_LERP_FUSED,+ MODEL_TENSOR.TIME_MIX_FIRST,+ MODEL_TENSOR.TIME_MIX_DECAY,+ MODEL_TENSOR.TIME_MIX_DECAY_W1,+ MODEL_TENSOR.TIME_MIX_DECAY_W2,+ MODEL_TENSOR.TIME_MIX_KEY,+ MODEL_TENSOR.TIME_MIX_VALUE,+ MODEL_TENSOR.TIME_MIX_RECEPTANCE,+ MODEL_TENSOR.TIME_MIX_GATE,+ MODEL_TENSOR.TIME_MIX_LN,+ MODEL_TENSOR.TIME_MIX_OUTPUT,+ MODEL_TENSOR.CHANNEL_MIX_LERP_K,+ MODEL_TENSOR.CHANNEL_MIX_LERP_R,+ MODEL_TENSOR.CHANNEL_MIX_KEY,+ MODEL_TENSOR.CHANNEL_MIX_RECEPTANCE,+ MODEL_TENSOR.CHANNEL_MIX_VALUE,+ ],+ MODEL_ARCH.RWKV6QWEN2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.TIME_MIX_W1,+ MODEL_TENSOR.TIME_MIX_W2,+ MODEL_TENSOR.TIME_MIX_LERP_X,+ MODEL_TENSOR.TIME_MIX_LERP_K,+ MODEL_TENSOR.TIME_MIX_LERP_V,+ MODEL_TENSOR.TIME_MIX_LERP_R,+ MODEL_TENSOR.TIME_MIX_LERP_G,+ MODEL_TENSOR.TIME_MIX_LERP_W,+ MODEL_TENSOR.TIME_MIX_LERP_FUSED,+ MODEL_TENSOR.TIME_MIX_FIRST,+ MODEL_TENSOR.TIME_MIX_DECAY,+ MODEL_TENSOR.TIME_MIX_DECAY_W1,+ MODEL_TENSOR.TIME_MIX_DECAY_W2,+ MODEL_TENSOR.TIME_MIX_KEY,+ MODEL_TENSOR.TIME_MIX_VALUE,+ MODEL_TENSOR.TIME_MIX_RECEPTANCE,+ MODEL_TENSOR.TIME_MIX_GATE,+ MODEL_TENSOR.TIME_MIX_LN,+ MODEL_TENSOR.TIME_MIX_OUTPUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.RWKV7: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_NORM_2,+ MODEL_TENSOR.TIME_MIX_LERP_FUSED,+ MODEL_TENSOR.TIME_MIX_W0,+ MODEL_TENSOR.TIME_MIX_W1,+ MODEL_TENSOR.TIME_MIX_W2,+ MODEL_TENSOR.TIME_MIX_A0,+ MODEL_TENSOR.TIME_MIX_A1,+ MODEL_TENSOR.TIME_MIX_A2,+ MODEL_TENSOR.TIME_MIX_V0,+ MODEL_TENSOR.TIME_MIX_V1,+ MODEL_TENSOR.TIME_MIX_V2,+ MODEL_TENSOR.TIME_MIX_G1,+ MODEL_TENSOR.TIME_MIX_G2,+ MODEL_TENSOR.TIME_MIX_K_K,+ MODEL_TENSOR.TIME_MIX_K_A,+ MODEL_TENSOR.TIME_MIX_R_K,+ MODEL_TENSOR.TIME_MIX_KEY,+ MODEL_TENSOR.TIME_MIX_VALUE,+ MODEL_TENSOR.TIME_MIX_RECEPTANCE,+ MODEL_TENSOR.TIME_MIX_LN,+ MODEL_TENSOR.TIME_MIX_OUTPUT,+ MODEL_TENSOR.CHANNEL_MIX_LERP_K,+ MODEL_TENSOR.CHANNEL_MIX_KEY,+ MODEL_TENSOR.CHANNEL_MIX_VALUE,+ ],+ MODEL_ARCH.ARWKV7: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.TIME_MIX_LERP_FUSED,+ MODEL_TENSOR.TIME_MIX_W0,+ MODEL_TENSOR.TIME_MIX_W1,+ MODEL_TENSOR.TIME_MIX_W2,+ MODEL_TENSOR.TIME_MIX_A0,+ MODEL_TENSOR.TIME_MIX_A1,+ MODEL_TENSOR.TIME_MIX_A2,+ MODEL_TENSOR.TIME_MIX_V0,+ MODEL_TENSOR.TIME_MIX_V1,+ MODEL_TENSOR.TIME_MIX_V2,+ MODEL_TENSOR.TIME_MIX_G1,+ MODEL_TENSOR.TIME_MIX_G2,+ MODEL_TENSOR.TIME_MIX_K_K,+ MODEL_TENSOR.TIME_MIX_K_A,+ MODEL_TENSOR.TIME_MIX_R_K,+ MODEL_TENSOR.TIME_MIX_KEY,+ MODEL_TENSOR.TIME_MIX_VALUE,+ MODEL_TENSOR.TIME_MIX_RECEPTANCE,+ MODEL_TENSOR.TIME_MIX_LN,+ MODEL_TENSOR.TIME_MIX_OUTPUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.MAMBA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.SSM_IN,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_X,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_D,+ MODEL_TENSOR.SSM_OUT,+ ],+ MODEL_ARCH.MAMBA2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.SSM_IN,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_D,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_OUT,+ ],+ MODEL_ARCH.MAPLE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.JAMBA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.SSM_IN,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_X,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_DT_NORM,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_B_NORM,+ MODEL_TENSOR.SSM_C_NORM,+ MODEL_TENSOR.SSM_D,+ MODEL_TENSOR.SSM_OUT,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.XVERSE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.COMMAND_R: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_Q_NORM,+ ],+ MODEL_ARCH.COHERE2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.COHERE2MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.DBRX: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_OUT_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.OLMO: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.OLMO2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.SEED_OSS: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ ],+ MODEL_ARCH.OLMOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ ],+ MODEL_ARCH.MUSE_GLIMMER: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ ],+ MODEL_ARCH.OPENELM: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.ARCTIC: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_NORM_EXP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.DEEPSEEK: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ ],+ MODEL_ARCH.DEEPSEEK2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ # NextN/MTP tensors+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.DEEPSEEK2OCR: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ ],+ MODEL_ARCH.DEEPSEEK32: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.INDEXER_K_NORM,+ MODEL_TENSOR.INDEXER_PROJ,+ MODEL_TENSOR.INDEXER_ATTN_K,+ MODEL_TENSOR.INDEXER_ATTN_Q_B,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.DEEPSEEK4: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.HC_HEAD_FN,+ MODEL_TENSOR.HC_HEAD_BASE,+ MODEL_TENSOR.HC_HEAD_SCALE,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV,+ MODEL_TENSOR.ATTN_KV_NORM,+ MODEL_TENSOR.ATTN_OUT_A,+ MODEL_TENSOR.ATTN_OUT_B,+ MODEL_TENSOR.HC_ATTN_FN,+ MODEL_TENSOR.HC_ATTN_BASE,+ MODEL_TENSOR.HC_ATTN_SCALE,+ MODEL_TENSOR.HC_FFN_FN,+ MODEL_TENSOR.HC_FFN_BASE,+ MODEL_TENSOR.HC_FFN_SCALE,+ MODEL_TENSOR.ATTN_COMPRESSOR_WKV,+ MODEL_TENSOR.ATTN_COMPRESSOR_WGATE,+ MODEL_TENSOR.ATTN_COMPRESSOR_APE,+ MODEL_TENSOR.ATTN_COMPRESSOR_NORM,+ MODEL_TENSOR.INDEXER_PROJ,+ MODEL_TENSOR.INDEXER_ATTN_Q_B,+ MODEL_TENSOR.INDEXER_COMPRESSOR_WKV,+ MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE,+ MODEL_TENSOR.INDEXER_COMPRESSOR_APE,+ MODEL_TENSOR.INDEXER_COMPRESSOR_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_TID2EID,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_EXP_PROBS_B_VL,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.ERNIE4_5_MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ ],+ MODEL_ARCH.PLM: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_DOWN,+ ],+ MODEL_ARCH.CHATGLM : [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.GLM4 : [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.GLM4_MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ # NextN/MTP tensors+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.GLM_DSA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.INDEXER_K_NORM,+ MODEL_TENSOR.INDEXER_PROJ,+ MODEL_TENSOR.INDEXER_ATTN_K,+ MODEL_TENSOR.INDEXER_ATTN_Q_B,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.GLM5_NEXT: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ # mHC+ MODEL_TENSOR.HC_ATTN_FN,+ MODEL_TENSOR.HC_ATTN_BASE,+ MODEL_TENSOR.HC_ATTN_SCALE,+ MODEL_TENSOR.HC_FFN_FN,+ MODEL_TENSOR.HC_FFN_BASE,+ MODEL_TENSOR.HC_FFN_SCALE,+ # KDA (linear attention) layers+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.SSM_CONV1D_Q,+ MODEL_TENSOR.SSM_CONV1D_K,+ MODEL_TENSOR.SSM_CONV1D_V,+ MODEL_TENSOR.SSM_F_A,+ MODEL_TENSOR.SSM_F_B,+ MODEL_TENSOR.SSM_BETA,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_G_A,+ MODEL_TENSOR.SSM_G_B,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ # MLA (nope) + DSA layers+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.INDEXER_K_NORM,+ MODEL_TENSOR.INDEXER_PROJ,+ MODEL_TENSOR.INDEXER_ATTN_K,+ MODEL_TENSOR.INDEXER_ATTN_Q_B,+ MODEL_TENSOR.INDEXER_KPOOL_GATE,+ MODEL_TENSOR.INDEXER_KPOOL_APE,+ # FFN+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.BITNET: [+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.ATTN_SUB_NORM,+ MODEL_TENSOR.FFN_SUB_NORM,+ ],+ MODEL_ARCH.T5: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.DEC_ATTN_NORM,+ MODEL_TENSOR.DEC_ATTN_Q,+ MODEL_TENSOR.DEC_ATTN_K,+ MODEL_TENSOR.DEC_ATTN_V,+ MODEL_TENSOR.DEC_ATTN_OUT,+ MODEL_TENSOR.DEC_ATTN_REL_B,+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM,+ MODEL_TENSOR.DEC_CROSS_ATTN_Q,+ MODEL_TENSOR.DEC_CROSS_ATTN_K,+ MODEL_TENSOR.DEC_CROSS_ATTN_V,+ MODEL_TENSOR.DEC_CROSS_ATTN_OUT,+ MODEL_TENSOR.DEC_CROSS_ATTN_REL_B,+ MODEL_TENSOR.DEC_FFN_NORM,+ MODEL_TENSOR.DEC_FFN_GATE,+ MODEL_TENSOR.DEC_FFN_DOWN,+ MODEL_TENSOR.DEC_FFN_UP,+ MODEL_TENSOR.DEC_OUTPUT_NORM,+ MODEL_TENSOR.ENC_ATTN_NORM,+ MODEL_TENSOR.ENC_ATTN_Q,+ MODEL_TENSOR.ENC_ATTN_K,+ MODEL_TENSOR.ENC_ATTN_V,+ MODEL_TENSOR.ENC_ATTN_OUT,+ MODEL_TENSOR.ENC_ATTN_REL_B,+ MODEL_TENSOR.ENC_FFN_NORM,+ MODEL_TENSOR.ENC_FFN_GATE,+ MODEL_TENSOR.ENC_FFN_DOWN,+ MODEL_TENSOR.ENC_FFN_UP,+ MODEL_TENSOR.ENC_OUTPUT_NORM,+ ],+ MODEL_ARCH.T5ENCODER: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ENC_ATTN_NORM,+ MODEL_TENSOR.ENC_ATTN_Q,+ MODEL_TENSOR.ENC_ATTN_K,+ MODEL_TENSOR.ENC_ATTN_V,+ MODEL_TENSOR.ENC_ATTN_OUT,+ MODEL_TENSOR.ENC_ATTN_REL_B,+ MODEL_TENSOR.ENC_FFN_NORM,+ MODEL_TENSOR.ENC_FFN_GATE,+ MODEL_TENSOR.ENC_FFN_DOWN,+ MODEL_TENSOR.ENC_FFN_UP,+ MODEL_TENSOR.ENC_OUTPUT_NORM,+ ],+ MODEL_ARCH.JAIS: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.JAIS2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.NEMOTRON: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.NEMOTRON_H: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.SSM_IN,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_D,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_OUT,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.NEMOTRON_H_MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.SSM_IN,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_D,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_OUT,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ # experts+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ # expert latent+ MODEL_TENSOR.MOE_LATENT_DOWN,+ MODEL_TENSOR.MOE_LATENT_UP,+ # shared expert+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ # NextN/MTP (draft head)+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.EXAONE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.EXAONE4: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_POST_NORM,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.EXAONE_MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.GRANITE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.GRANITE_MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ ],+ MODEL_ARCH.GRANITE_HYBRID: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.SSM_IN,+ MODEL_TENSOR.SSM_CONV1D,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_D,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.SSM_OUT,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ # MoE+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ # Dense+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.GRANITE_SWITCH: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.GRANITE_SWA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ # MoE (GraniteMoeSWA)+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ # Shared expert - gate+up kept fused in FFN_UP_SHEXP (LLM_FFN_SWIGLU)+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ ],+ MODEL_ARCH.CHAMELEON: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.WAVTOKENIZER_DEC: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.CONV1D,+ MODEL_TENSOR.CONVNEXT_DW,+ MODEL_TENSOR.CONVNEXT_NORM,+ MODEL_TENSOR.CONVNEXT_PW1,+ MODEL_TENSOR.CONVNEXT_PW2,+ MODEL_TENSOR.CONVNEXT_GAMMA,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.POSNET_CONV1,+ MODEL_TENSOR.POSNET_CONV2,+ MODEL_TENSOR.POSNET_NORM,+ MODEL_TENSOR.POSNET_NORM1,+ MODEL_TENSOR.POSNET_NORM2,+ MODEL_TENSOR.POSNET_ATTN_NORM,+ MODEL_TENSOR.POSNET_ATTN_Q,+ MODEL_TENSOR.POSNET_ATTN_K,+ MODEL_TENSOR.POSNET_ATTN_V,+ MODEL_TENSOR.POSNET_ATTN_OUT,+ ],+ MODEL_ARCH.BAILINGMOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ ],+ MODEL_ARCH.BAILINGMOE2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ MODEL_TENSOR.LAYER_OUT_NORM,+ ],+ MODEL_ARCH.BAILINGMOE3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.SSM_CONV1D_Q,+ MODEL_TENSOR.SSM_CONV1D_K,+ MODEL_TENSOR.SSM_CONV1D_V,+ MODEL_TENSOR.SSM_F_A,+ MODEL_TENSOR.SSM_BETA,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_G_A,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.LAYER_OUT_NORM,+ ],+ MODEL_ARCH.DOTS1: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ ],+ MODEL_ARCH.DOTS3NOTE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.INDEXER_K_NORM,+ MODEL_TENSOR.INDEXER_PROJ,+ MODEL_TENSOR.INDEXER_ATTN_K,+ MODEL_TENSOR.INDEXER_ATTN_Q_B,+ # NextN/MTP tensors - preserved but unused+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.ARCEE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.AFMOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_PRE_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ ],+ MODEL_ARCH.LAGUNA: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ ],+ MODEL_ARCH.ERNIE4_5: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.PADDLEOCR: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.FALCON_H1: [+ # Token embedding+ MODEL_TENSOR.TOKEN_EMBD,++ # Input layernorm+ MODEL_TENSOR.ATTN_NORM,++ # Attention components+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q, # Query projection+ MODEL_TENSOR.ATTN_K, # Key projection+ MODEL_TENSOR.ATTN_V, # Value projection+ MODEL_TENSOR.ATTN_OUT, # Output projection++ # SSM components (Mamba2 specific)+ MODEL_TENSOR.SSM_IN, # Input projection for SSM+ MODEL_TENSOR.SSM_CONV1D, # Convolution layer+ MODEL_TENSOR.SSM_DT, # Delta time projection+ MODEL_TENSOR.SSM_A, # A parameter (log form)+ MODEL_TENSOR.SSM_D, # D parameter+ MODEL_TENSOR.SSM_NORM, # Normalization in SSM+ MODEL_TENSOR.SSM_OUT, # Output projection++ # Pre-feedforward layernorm+ MODEL_TENSOR.FFN_PRE_NORM,++ # Feed-forward network components+ MODEL_TENSOR.FFN_GATE, # Gate projection (SwiGLU)+ MODEL_TENSOR.FFN_DOWN, # Down projection+ MODEL_TENSOR.FFN_UP, # Up projection++ # Post-feedforward layernorm+ MODEL_TENSOR.OUTPUT_NORM, # Final layer norm+ MODEL_TENSOR.OUTPUT, # Output projection (lm_head)+ ],+ MODEL_ARCH.HUNYUAN_MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ ],+ MODEL_ARCH.HUNYUAN_DENSE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.HUNYUAN_VL: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.HY_V3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ # NextN/MTP tensors (draft head)+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.HY_V4: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.HC_HEAD_FN,+ MODEL_TENSOR.HC_HEAD_BASE,+ MODEL_TENSOR.HC_HEAD_SCALE,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.INDEXER_K_NORM,+ MODEL_TENSOR.INDEXER_PROJ,+ MODEL_TENSOR.INDEXER_ATTN_K,+ MODEL_TENSOR.INDEXER_ATTN_Q_B,+ MODEL_TENSOR.HC_ATTN_FN,+ MODEL_TENSOR.HC_ATTN_BASE,+ MODEL_TENSOR.HC_ATTN_SCALE,+ MODEL_TENSOR.HC_FFN_FN,+ MODEL_TENSOR.HC_FFN_BASE,+ MODEL_TENSOR.HC_FFN_SCALE,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ ],+ MODEL_ARCH.SMOLLM3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.GPT_OSS: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.LFM2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.SHORTCONV_CONV,+ MODEL_TENSOR.SHORTCONV_INPROJ,+ MODEL_TENSOR.SHORTCONV_OUTPROJ,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.ATTN_NORM, # operator_norm+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.DENSE_2_OUT, # LFM2-ColBert-350M+ MODEL_TENSOR.TOKEN_TYPES, # decision head+ MODEL_TENSOR.CLS,+ MODEL_TENSOR.CLS_NORM,+ MODEL_TENSOR.CLS_OUT,+ ],+ MODEL_ARCH.LFM2MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.TOKEN_EMBD_NORM,+ MODEL_TENSOR.SHORTCONV_CONV,+ MODEL_TENSOR.SHORTCONV_INPROJ,+ MODEL_TENSOR.SHORTCONV_OUTPROJ,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.ATTN_NORM, # operator_norm+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ ],+ MODEL_ARCH.SMALLTHINKER: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.APERTUS: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.LLADA_MOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ ],+ MODEL_ARCH.GROVEMOE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_CHEXP,+ MODEL_TENSOR.FFN_DOWN_CHEXP,+ MODEL_TENSOR.FFN_UP_CHEXP,+ ],+ MODEL_ARCH.MINIMAX01: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_NORM_2,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.MINIMAXM2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ ],+ MODEL_ARCH.MINIMAXM3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.INDEXER_Q_PROJ,+ MODEL_TENSOR.INDEXER_K_PROJ,+ MODEL_TENSOR.INDEXER_Q_NORM,+ MODEL_TENSOR.INDEXER_K_NORM,+ ],+ MODEL_ARCH.COGVLM: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.VISEXP_ATTN_QKV,+ MODEL_TENSOR.VISEXP_ATTN_OUT,+ MODEL_TENSOR.VISEXP_GATE,+ MODEL_TENSOR.VISEXP_UP,+ MODEL_TENSOR.VISEXP_DOWN,+ ],+ MODEL_ARCH.RND1: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.PANGU_EMBED: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.MISTRAL3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.EAGLE3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_NORM_2,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FC,+ MODEL_TENSOR.ENC_OUTPUT_NORM,+ MODEL_TENSOR.D2T,+ ],+ MODEL_ARCH.DFLASH: [+ MODEL_TENSOR.ATTN_POST_NORM,+ MODEL_TENSOR.FFN_POST_NORM,+ MODEL_TENSOR.LAYER_OUT_SCALE,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV,+ MODEL_TENSOR.ATTN_KV_NORM,+ MODEL_TENSOR.ATTN_OUT_A,+ MODEL_TENSOR.ATTN_OUT_B,+ MODEL_TENSOR.HC_ATTN_FN,+ MODEL_TENSOR.HC_ATTN_BASE,+ MODEL_TENSOR.HC_ATTN_SCALE,+ MODEL_TENSOR.HC_FFN_FN,+ MODEL_TENSOR.HC_FFN_BASE,+ MODEL_TENSOR.HC_FFN_SCALE,+ MODEL_TENSOR.HC_HEAD_FN,+ MODEL_TENSOR.HC_HEAD_BASE,+ MODEL_TENSOR.HC_HEAD_SCALE,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FC,+ MODEL_TENSOR.ENC_OUTPUT_NORM,+ MODEL_TENSOR.D2T,+ # optional DSpark heads+ MODEL_TENSOR.DSPARK_MARKOV_W1,+ MODEL_TENSOR.DSPARK_MARKOV_W2,+ MODEL_TENSOR.DSPARK_CONF_PROJ,+ MODEL_TENSOR.DFLASH_ATTN_CONV_BASE,+ MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ,+ MODEL_TENSOR.DFLASH_FFN_CONV_BASE,+ MODEL_TENSOR.DFLASH_FFN_CONV_PROJ,+ MODEL_TENSOR.DFLASH_SELECTOR_PREV,+ MODEL_TENSOR.DFLASH_SELECTOR_NEXT,+ MODEL_TENSOR.DFLASH_SELECTOR_HIDDEN,+ ],+ MODEL_ARCH.MISTRAL4: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ ],+ MODEL_ARCH.MIMO2: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_SINKS,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.LAYER_OUT_NORM,+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.STEP35: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ # NextN/MTP tensors (Step3p5 draft head)+ MODEL_TENSOR.NEXTN_EH_PROJ,+ MODEL_TENSOR.NEXTN_EMBED_TOKENS,+ MODEL_TENSOR.NEXTN_ENORM,+ MODEL_TENSOR.NEXTN_HNORM,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,+ MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,+ ],+ MODEL_ARCH.SPARK2_5: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.LLAMA_EMBED: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.MAINCODER: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.KIMI_LINEAR: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_QKV,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.SSM_CONV1D_Q,+ MODEL_TENSOR.SSM_CONV1D_K,+ MODEL_TENSOR.SSM_CONV1D_V,+ MODEL_TENSOR.SSM_F_A,+ MODEL_TENSOR.SSM_F_B,+ MODEL_TENSOR.SSM_BETA,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_G_A,+ MODEL_TENSOR.SSM_G_B,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ ],+ MODEL_ARCH.KIMI_K3: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.OUTPUT_RES_SCORE,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_RES_SCORE,+ MODEL_TENSOR.FFN_RES_SCORE,+ # MLA (full-attention layers)+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.ATTN_Q_A,+ MODEL_TENSOR.ATTN_Q_B,+ MODEL_TENSOR.ATTN_KV_A_MQA,+ MODEL_TENSOR.ATTN_KV_B,+ MODEL_TENSOR.ATTN_K_B,+ MODEL_TENSOR.ATTN_V_B,+ MODEL_TENSOR.ATTN_Q_A_NORM,+ MODEL_TENSOR.ATTN_KV_A_NORM,+ # KDA (linear-attention layers)+ MODEL_TENSOR.SSM_CONV1D_Q,+ MODEL_TENSOR.SSM_CONV1D_K,+ MODEL_TENSOR.SSM_CONV1D_V,+ MODEL_TENSOR.SSM_F_A,+ MODEL_TENSOR.SSM_F_B,+ MODEL_TENSOR.SSM_BETA,+ MODEL_TENSOR.SSM_A,+ MODEL_TENSOR.SSM_G,+ MODEL_TENSOR.SSM_DT,+ MODEL_TENSOR.SSM_NORM,+ # FFN+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_ROUTED_DOWN,+ MODEL_TENSOR.FFN_ROUTED_UP,+ MODEL_TENSOR.FFN_ROUTED_NORM,+ ],+ MODEL_ARCH.TALKIE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.LAYER_OUT_SCALE,+ ],+ MODEL_ARCH.MELLUM: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ ],+ MODEL_ARCH.NANBEIGE: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.QWEN3TTS: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.POCKETTTS: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_UP,+ ],+ MODEL_ARCH.K2HORIZON: [+ MODEL_TENSOR.TOKEN_EMBD,+ MODEL_TENSOR.OUTPUT_NORM,+ MODEL_TENSOR.OUTPUT,+ MODEL_TENSOR.ATTN_NORM,+ MODEL_TENSOR.ATTN_Q,+ MODEL_TENSOR.ATTN_Q_NORM,+ MODEL_TENSOR.ATTN_K,+ MODEL_TENSOR.ATTN_K_NORM,+ MODEL_TENSOR.ATTN_V,+ MODEL_TENSOR.ATTN_V_GATE,+ MODEL_TENSOR.ATTN_V_EXP,+ MODEL_TENSOR.ATTN_OUT,+ MODEL_TENSOR.ATTN_GATE,+ MODEL_TENSOR.FFN_NORM,+ MODEL_TENSOR.FFN_GATE,+ MODEL_TENSOR.FFN_UP,+ MODEL_TENSOR.FFN_DOWN,+ MODEL_TENSOR.FFN_GATE_INP,+ MODEL_TENSOR.FFN_EXP_PROBS_B,+ MODEL_TENSOR.FFN_GATE_EXP,+ MODEL_TENSOR.FFN_UP_EXP,+ MODEL_TENSOR.FFN_DOWN_EXP,+ MODEL_TENSOR.FFN_GATE_SHEXP,+ MODEL_TENSOR.FFN_UP_SHEXP,+ MODEL_TENSOR.FFN_DOWN_SHEXP,+ ],+}++# tensors that will not be serialized+MODEL_TENSOR_SKIP: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {+ MODEL_ARCH.LLAMA: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.DECI: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.BAICHUAN: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.QWEN: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.CODESHELL: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.ORION: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.STARCODER2: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.XVERSE: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.DEEPSEEK: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.DEEPSEEK2: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.DEEPSEEK2OCR: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.DEEPSEEK32: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.HY_V4: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.CHATGLM: [+ MODEL_TENSOR.ROPE_FREQS,+ ],+ MODEL_ARCH.NEMOTRON: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.BAILINGMOE: [+ MODEL_TENSOR.ROPE_FREQS,+ ],+ MODEL_ARCH.PANGU_EMBED: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+ MODEL_ARCH.NANBEIGE: [+ MODEL_TENSOR.ROPE_FREQS,+ MODEL_TENSOR.ATTN_ROT_EMBD,+ ],+}++#+# types+#+++class TokenType(IntEnum):+ NORMAL = 1+ UNKNOWN = 2+ CONTROL = 3+ USER_DEFINED = 4+ UNUSED = 5+ BYTE = 6+++class RopeScalingType(Enum):+ NONE = 'none'+ LINEAR = 'linear'+ YARN = 'yarn'+ LONGROPE = 'longrope'+++# M-RoPE: input position slot (t, y, x, z) that feeds each RoPE section, in section order+class RopeSectionOrder(Enum):+ TYXZ = 'tyxz' # default+ ZYXT = 'zyxt'+++class PoolingType(IntEnum):+ NONE = 0+ MEAN = 1+ CLS = 2+ LAST = 3+ RANK = 4+++class GGMLQuantizationType(IntEnum):+ F32 = 0+ F16 = 1+ Q4_0 = 2+ Q4_1 = 3+ Q5_0 = 6+ Q5_1 = 7+ Q8_0 = 8+ Q8_1 = 9+ Q2_K = 10+ Q3_K = 11+ Q4_K = 12+ Q5_K = 13+ Q6_K = 14+ Q8_K = 15+ IQ2_XXS = 16+ IQ2_XS = 17+ IQ3_XXS = 18+ IQ1_S = 19+ IQ4_NL = 20+ IQ3_S = 21+ IQ2_S = 22+ IQ4_XS = 23+ I8 = 24+ I16 = 25+ I32 = 26+ I64 = 27+ F64 = 28+ IQ1_M = 29+ BF16 = 30+ TQ1_0 = 34+ TQ2_0 = 35+ MXFP4 = 39+ NVFP4 = 40+ Q1_0 = 41+ Q2_0 = 42+++class ExpertGatingFuncType(IntEnum):+ SOFTMAX = 1+ SIGMOID = 2+ SQRTSOFTPLUS = 4+++# TODO: add GGMLFileType from ggml_ftype in ggml.h+++# from llama_ftype in llama.h+# ALL VALUES SHOULD BE THE SAME HERE AS THEY ARE OVER THERE.+class LlamaFileType(IntEnum):+ ALL_F32 = 0+ MOSTLY_F16 = 1 # except 1d tensors+ MOSTLY_Q4_0 = 2 # except 1d tensors+ MOSTLY_Q4_1 = 3 # except 1d tensors+ # MOSTLY_Q4_1_SOME_F16 = 4 # tok_embeddings.weight and output.weight are F16+ # MOSTLY_Q4_2 = 5 # support has been removed+ # MOSTLY_Q4_3 = 6 # support has been removed+ MOSTLY_Q8_0 = 7 # except 1d tensors+ MOSTLY_Q5_0 = 8 # except 1d tensors+ MOSTLY_Q5_1 = 9 # except 1d tensors+ MOSTLY_Q2_K = 10 # except 1d tensors+ MOSTLY_Q3_K_S = 11 # except 1d tensors+ MOSTLY_Q3_K_M = 12 # except 1d tensors+ MOSTLY_Q3_K_L = 13 # except 1d tensors+ MOSTLY_Q4_K_S = 14 # except 1d tensors+ MOSTLY_Q4_K_M = 15 # except 1d tensors+ MOSTLY_Q5_K_S = 16 # except 1d tensors+ MOSTLY_Q5_K_M = 17 # except 1d tensors+ MOSTLY_Q6_K = 18 # except 1d tensors+ MOSTLY_IQ2_XXS = 19 # except 1d tensors+ MOSTLY_IQ2_XS = 20 # except 1d tensors+ MOSTLY_Q2_K_S = 21 # except 1d tensors+ MOSTLY_IQ3_XS = 22 # except 1d tensors+ MOSTLY_IQ3_XXS = 23 # except 1d tensors+ MOSTLY_IQ1_S = 24 # except 1d tensors+ MOSTLY_IQ4_NL = 25 # except 1d tensors+ MOSTLY_IQ3_S = 26 # except 1d tensors+ MOSTLY_IQ3_M = 27 # except 1d tensors+ MOSTLY_IQ2_S = 28 # except 1d tensors+ MOSTLY_IQ2_M = 29 # except 1d tensors+ MOSTLY_IQ4_XS = 30 # except 1d tensors+ MOSTLY_IQ1_M = 31 # except 1d tensors+ MOSTLY_BF16 = 32 # except 1d tensors+ # MOSTLY_Q4_0_4_4 = 33 # removed from gguf files, use Q4_0 and runtime repack+ # MOSTLY_Q4_0_4_8 = 34 # removed from gguf files, use Q4_0 and runtime repack+ # MOSTLY_Q4_0_8_8 = 35 # removed from gguf files, use Q4_0 and runtime repack+ MOSTLY_TQ1_0 = 36 # except 1d tensors+ MOSTLY_TQ2_0 = 37 # except 1d tensors+ MOSTLY_MXFP4_MOE = 38 # except 1d tensors+ MOSTLY_NVFP4 = 39 # except 1d tensors+ MOSTLY_Q1_0 = 40 # except 1d tensors+ MOSTLY_Q2_0 = 41 # except 1d tensors++ GUESSED = 1024 # not specified in the model file+++class GGUFEndian(IntEnum):+ LITTLE = 0+ BIG = 1+++class GGUFValueType(IntEnum):+ UINT8 = 0+ INT8 = 1+ UINT16 = 2+ INT16 = 3+ UINT32 = 4+ INT32 = 5+ FLOAT32 = 6+ BOOL = 7+ STRING = 8+ ARRAY = 9+ UINT64 = 10+ INT64 = 11+ FLOAT64 = 12++ @staticmethod+ def get_type(val: Any) -> GGUFValueType:+ if isinstance(val, (str, bytes, bytearray)):+ return GGUFValueType.STRING+ elif isinstance(val, list):+ return GGUFValueType.ARRAY+ elif isinstance(val, float):+ return GGUFValueType.FLOAT32+ elif isinstance(val, bool):+ return GGUFValueType.BOOL+ elif isinstance(val, int):+ return GGUFValueType.INT32+ # TODO: need help with 64-bit types in Python+ else:+ raise ValueError(f"Unknown type: {type(val)}")+++class DecisionType:+ LAYA = "laya" # head blocks + scorer on the hidden state of one marker token per option+ OPENJEV = "openjev" # logits of one label token per option+ LEV = "lev" # same as openjev, noul is read from a rating scale+ KEV = "kev" # dot product of the hidden states of the last token and of one end token per option+ NIMBLE = "nimble" # same as openjev, the prompt lists all the questions of the request+ CLEF = "clef" # joint head over all questions, one score per option+ PPLX_DECIDER = "pplx-decider" # same as openjev, label codes of 1 or 2 letters+ VLLM_DECISION2 = "vllm-decision2" # head over the end token of each option and the last token, one score per option+ VLLM_D3 = "vllm-d3" # same as pplx-decider, non-causal attention+ LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type+ LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout+++class VisionProjectorType:+ GEMMA3 = "gemma3"+ GEMMA3NV = "gemma3nv"+ GEMMA3NA = "gemma3na"+ GEMMA4V = "gemma4v"+ GEMMA4A = "gemma4a"+ GEMMA4UV = "gemma4uv" # "unified" variant+ GEMMA4UA = "gemma4ua" # "unified" variant+ PHI4 = "phi4"+ IDEFICS3 = "idefics3"+ PIXTRAL = "pixtral"+ LLAMA4 = "llama4"+ QWEN2VL = "qwen2vl_merger"+ QWEN25VL = "qwen2.5vl_merger"+ EXAONE4_5 = "exaone4_5"+ QWEN3VL = "qwen3vl_merger"+ LING3VL = "ling3vl"+ STEP3VL = "step3vl"+ ULTRAVOX = "ultravox"+ INTERNVL = "internvl"+ QWEN2A = "qwen2a" # audio+ QWEN3A = "qwen3a" # audio+ GLMA = "glma" # audio+ QWEN25O = "qwen2.5o" # omni+ VOXTRAL = "voxtral"+ MERALION = "meralion" # audio: Whisper + gated MLP adaptor+ LFM2 = "lfm2"+ KIMIVL = "kimivl"+ PADDLEOCR = "paddleocr"+ KIMIK25 = "kimik25"+ LIGHTONOCR = "lightonocr"+ COGVLM = "cogvlm"+ JANUS_PRO = "janus_pro"+ DOTSOCR = "dots_ocr"+ DOTS3NOTE_V = "dots3note_v"+ DOTS3NOTE_A = "dots3note_a" # audio+ DEEPSEEKOCR = "deepseekocr"+ DEEPSEEKOCR2 = "deepseekocr2"+ DEEPSEEK4V = "deepseek4v"+ LFM2A = "lfm2a" # audio+ MUSIC_FLAMINGO = "musicflamingo" # audio+ GLM4V = "glm4v"+ GLM5V = "glm5v"+ YOUTUVL = "youtuvl"+ NEMOTRON_V2_VL = "nemotron_v2_vl"+ QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder+ QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor+ POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder+ POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder+ HUNYUANVL = "hunyuanvl"+ PARAKEET = "parakeet" # audio+ MINIMAXM3 = "minimax_m3"+ MINICPMV4_6 = "minicpmv4_6"+ MINICPMV4_7 = "minicpmv4_7"+ GRANITE_SPEECH = "granite_speech" # audio+ MIMOVL = "mimovl"+ MIMO_AUDIO = "mimo_audio"+ GRANITE4_VISION = "granite4_vision"+ MUSE_GLIMMER = "muse-glimmer"+ COHERE2V = "cohere2v"+ D1OMNI_V = "d1omni_v" # lfm2 vision, without separator tokens+ D1OMNI_A = "d1omni_a" # lfm2a audio, with a residual block after the projector+++# Items here are (block size, type size)+QK_K = 256+GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = {+ GGMLQuantizationType.F32: (1, 4),+ GGMLQuantizationType.F16: (1, 2),+ GGMLQuantizationType.Q4_0: (32, 2 + 16),+ GGMLQuantizationType.Q4_1: (32, 2 + 2 + 16),+ GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16),+ GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16),+ GGMLQuantizationType.Q8_0: (32, 2 + 32),+ GGMLQuantizationType.Q8_1: (32, 2 + 2 + 32),+ GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4),+ GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12),+ GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12),+ GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),+ GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),+ GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8),+ GGMLQuantizationType.IQ2_XXS: (256, 2 + QK_K // 4),+ GGMLQuantizationType.IQ2_XS: (256, 2 + QK_K // 4 + QK_K // 32),+ GGMLQuantizationType.IQ3_XXS: (256, 2 + QK_K // 4 + QK_K // 8),+ GGMLQuantizationType.IQ1_S: (256, 2 + QK_K // 8 + QK_K // 16),+ GGMLQuantizationType.IQ4_NL: (32, 2 + 16),+ GGMLQuantizationType.IQ3_S: (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4),+ GGMLQuantizationType.IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16),+ GGMLQuantizationType.IQ4_XS: (256, 2 + 2 + QK_K // 2 + QK_K // 64),+ GGMLQuantizationType.I8: (1, 1),+ GGMLQuantizationType.I16: (1, 2),+ GGMLQuantizationType.I32: (1, 4),+ GGMLQuantizationType.I64: (1, 8),+ GGMLQuantizationType.F64: (1, 8),+ GGMLQuantizationType.IQ1_M: (256, QK_K // 8 + QK_K // 16 + QK_K // 32),+ GGMLQuantizationType.BF16: (1, 2),+ GGMLQuantizationType.TQ1_0: (256, 2 + 4 * 13),+ GGMLQuantizationType.TQ2_0: (256, 2 + 64),+ GGMLQuantizationType.MXFP4: (32, 1 + 16),+ GGMLQuantizationType.NVFP4: (64, 4 + 32),+ GGMLQuantizationType.Q1_0: (128, 2 + 16),+ GGMLQuantizationType.Q2_0: (64, 2 + 16),+}+++# Aliases for backward compatibility.++# general+KEY_GENERAL_ARCHITECTURE = Keys.General.ARCHITECTURE+KEY_GENERAL_QUANTIZATION_VERSION = Keys.General.QUANTIZATION_VERSION+KEY_GENERAL_ALIGNMENT = Keys.General.ALIGNMENT+KEY_GENERAL_NAME = Keys.General.NAME+KEY_GENERAL_AUTHOR = Keys.General.AUTHOR+KEY_GENERAL_URL = Keys.General.URL+KEY_GENERAL_DESCRIPTION = Keys.General.DESCRIPTION+KEY_GENERAL_LICENSE = Keys.General.LICENSE+KEY_GENERAL_SOURCE_URL = Keys.General.SOURCE_URL+KEY_GENERAL_FILE_TYPE = Keys.General.FILE_TYPE++# LLM+KEY_VOCAB_SIZE = Keys.LLM.VOCAB_SIZE+KEY_CONTEXT_LENGTH = Keys.LLM.CONTEXT_LENGTH+KEY_EMBEDDING_LENGTH = Keys.LLM.EMBEDDING_LENGTH+KEY_BLOCK_COUNT = Keys.LLM.BLOCK_COUNT+KEY_FEED_FORWARD_LENGTH = Keys.LLM.FEED_FORWARD_LENGTH+KEY_USE_PARALLEL_RESIDUAL = Keys.LLM.USE_PARALLEL_RESIDUAL+KEY_TENSOR_DATA_LAYOUT = Keys.LLM.TENSOR_DATA_LAYOUT++# attention+KEY_ATTENTION_HEAD_COUNT = Keys.Attention.HEAD_COUNT+KEY_ATTENTION_HEAD_COUNT_KV = Keys.Attention.HEAD_COUNT_KV+KEY_ATTENTION_MAX_ALIBI_BIAS = Keys.Attention.MAX_ALIBI_BIAS+KEY_ATTENTION_CLAMP_KQV = Keys.Attention.CLAMP_KQV+KEY_ATTENTION_LAYERNORM_EPS = Keys.Attention.LAYERNORM_EPS+KEY_ATTENTION_LAYERNORM_RMS_EPS = Keys.Attention.LAYERNORM_RMS_EPS++# RoPE+KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT+KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE+KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE+KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR+KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN+KEY_ROPE_SCALING_FINETUNED = Keys.Rope.SCALING_FINETUNED++# SSM+KEY_SSM_CONV_KERNEL = Keys.SSM.CONV_KERNEL+KEY_SSM_INNER_SIZE = Keys.SSM.INNER_SIZE+KEY_SSM_STATE_SIZE = Keys.SSM.STATE_SIZE+KEY_SSM_TIME_STEP_RANK = Keys.SSM.TIME_STEP_RANK+KEY_SSM_GROUP_COUNT = Keys.SSM.GROUP_COUNT+KEY_SSM_DT_B_C_RMS = Keys.SSM.DT_B_C_RMS++# KDA+KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM+KEY_KDA_SAFE_GATE = Keys.KDA.SAFE_GATE+KEY_KDA_GATE_LOWER_BOUND = Keys.KDA.GATE_LOWER_BOUND++# tokenization+KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL+KEY_TOKENIZER_PRE = Keys.Tokenizer.PRE+KEY_TOKENIZER_LIST = Keys.Tokenizer.LIST+KEY_TOKENIZER_TOKEN_TYPE = Keys.Tokenizer.TOKEN_TYPE+KEY_TOKENIZER_SCORES = Keys.Tokenizer.SCORES+KEY_TOKENIZER_MERGES = Keys.Tokenizer.MERGES+KEY_TOKENIZER_BOS_ID = Keys.Tokenizer.BOS_ID+KEY_TOKENIZER_EOS_ID = Keys.Tokenizer.EOS_ID+KEY_TOKENIZER_EOT_ID = Keys.Tokenizer.EOT_ID+KEY_TOKENIZER_EOM_ID = Keys.Tokenizer.EOM_ID+KEY_TOKENIZER_UNK_ID = Keys.Tokenizer.UNK_ID+KEY_TOKENIZER_SEP_ID = Keys.Tokenizer.SEP_ID+KEY_TOKENIZER_PAD_ID = Keys.Tokenizer.PAD_ID+KEY_TOKENIZER_MASK_ID = Keys.Tokenizer.MASK_ID+KEY_TOKENIZER_HF_JSON = Keys.Tokenizer.HF_JSON+KEY_TOKENIZER_RWKV = Keys.Tokenizer.RWKV++KEY_TOKENIZER_FIM_PRE_ID = Keys.Tokenizer.FIM_PRE_ID+KEY_TOKENIZER_FIM_SUF_ID = Keys.Tokenizer.FIM_SUF_ID+KEY_TOKENIZER_FIM_MID_ID = Keys.Tokenizer.FIM_MID_ID+KEY_TOKENIZER_FIM_PAD_ID = Keys.Tokenizer.FIM_PAD_ID+KEY_TOKENIZER_FIM_REP_ID = Keys.Tokenizer.FIM_REP_ID+KEY_TOKENIZER_FIM_SEP_ID = Keys.Tokenizer.FIM_SEP_ID++# deprecated+KEY_TOKENIZER_PREFIX_ID = Keys.Tokenizer.PREFIX_ID+KEY_TOKENIZER_SUFFIX_ID = Keys.Tokenizer.SUFFIX_ID+KEY_TOKENIZER_MIDDLE_ID = Keys.Tokenizer.MIDDLE_IDdiff --git a/python/freetoken/third_party/gguf/freetoken.patch b/python/freetoken/third_party/gguf/freetoken.patchnew file mode 100644index 00000000..b8016ff5--- /dev/null+++ b/python/freetoken/third_party/gguf/freetoken.patch@@ -0,0 +1,23 @@+--- a/gguf-py/gguf/__init__.py++++ b/gguf-py/gguf/__init__.py+@@ -1,9 +1,4 @@+ from .constants import *+ from .lazy import *+ from .gguf_reader import *+-from .gguf_writer import *+ from .quants import *+-from .tensor_mapping import *+-from .vocab import *+-from .utility import *+-from .metadata import *+--- a/gguf-py/gguf/gguf_reader.py++++ b/gguf-py/gguf/gguf_reader.py+@@ -21,7 +21,7 @@+ # Allow running file in package as a script.+ sys.path.insert(0, str(Path(__file__).parent.parent))++-from gguf.constants import (++from .constants import (+ GGML_MAX_DIMS,+ GGML_QUANT_SIZES,+ GGUF_DEFAULT_ALIGNMENT,diff --git a/python/freetoken/third_party/gguf/gguf_reader.py b/python/freetoken/third_party/gguf/gguf_reader.pynew file mode 100644index 00000000..41eb6521--- /dev/null+++ b/python/freetoken/third_party/gguf/gguf_reader.py@@ -0,0 +1,390 @@+#+# GGUF file reading/modification support. For API usage information,+# please see the files scripts/ for some fairly simple examples.+#+from __future__ import annotations++import logging+import os+import sys+from collections import OrderedDict+from typing import Any, Literal, NamedTuple, TypeVar, Union++import numpy as np+import numpy.typing as npt++from .quants import quant_shape_to_byte_shape++if __name__ == "__main__":+ from pathlib import Path++ # Allow running file in package as a script.+ sys.path.insert(0, str(Path(__file__).parent.parent))++from .constants import (+ GGML_MAX_DIMS,+ GGML_QUANT_SIZES,+ GGUF_DEFAULT_ALIGNMENT,+ GGUF_MAGIC,+ GGUF_VERSION,+ GGMLQuantizationType,+ GGUFValueType,+ GGUFEndian,+)++# limits mirroring ggml/src/gguf.cpp (not part of gguf.h)+GGUF_MAX_STRING_LENGTH = 1024 * 1024 * 1024+GGUF_MAX_ARRAY_ELEMENTS = 1024 * 1024 * 1024++logger = logging.getLogger(__name__)++READER_SUPPORTED_VERSIONS = [2, GGUF_VERSION]+++class ReaderField(NamedTuple):+ # Offset to start of this field.+ offset: int++ # Name of the field (not necessarily from file data).+ name: str++ # Data parts. Some types have multiple components, such as strings+ # that consist of a length followed by the string data.+ parts: list[npt.NDArray[Any]] = []++ # Indexes into parts that we can call the actual data. For example+ # an array of strings will be populated with indexes to the actual+ # string data.+ data: list[int] = [-1]++ types: list[GGUFValueType] = []++ def contents(self, index_or_slice: int | slice = slice(None)) -> Any:+ if self.types:+ to_string = lambda x: str(x.tobytes(), encoding='utf-8') # noqa: E731+ main_type = self.types[0]++ if main_type == GGUFValueType.ARRAY:+ sub_type = self.types[-1]++ if sub_type == GGUFValueType.STRING:+ indices = self.data[index_or_slice]++ if isinstance(index_or_slice, int):+ return to_string(self.parts[indices]) # type: ignore+ else:+ return [to_string(self.parts[idx]) for idx in indices] # type: ignore+ else:+ # FIXME: When/if _get_field_parts() support multi-dimensional arrays, this must do so too++ # Check if it's unsafe to perform slice optimization on data+ # if any(True for idx in self.data if len(self.parts[idx]) != 1):+ # optim_slice = slice(None)+ # else:+ # optim_slice = index_or_slice+ # index_or_slice = slice(None)++ # if isinstance(optim_slice, int):+ # return self.parts[self.data[optim_slice]].tolist()[0]+ # else:+ # return [pv for idx in self.data[optim_slice] for pv in self.parts[idx].tolist()][index_or_slice]++ if isinstance(index_or_slice, int):+ return self.parts[self.data[index_or_slice]].tolist()[0]+ else:+ return [pv for idx in self.data[index_or_slice] for pv in self.parts[idx].tolist()]++ if main_type == GGUFValueType.STRING:+ return to_string(self.parts[-1])+ else:+ return self.parts[-1].tolist()[0]++ return None+++class ReaderTensor(NamedTuple):+ name: str+ tensor_type: GGMLQuantizationType+ shape: npt.NDArray[np.uint32]+ n_elements: int+ n_bytes: int+ data_offset: int+ data: npt.NDArray[Any]+ field: ReaderField+++class GGUFReader:+ # I - same as host, S - swapped+ byte_order: Literal['I', 'S'] = 'I'+ alignment: int = GGUF_DEFAULT_ALIGNMENT+ data_offset: int++ # Note: Internal helper, API may change.+ gguf_scalar_to_np: dict[GGUFValueType, type[np.generic]] = {+ GGUFValueType.UINT8: np.uint8,+ GGUFValueType.INT8: np.int8,+ GGUFValueType.UINT16: np.uint16,+ GGUFValueType.INT16: np.int16,+ GGUFValueType.UINT32: np.uint32,+ GGUFValueType.INT32: np.int32,+ GGUFValueType.FLOAT32: np.float32,+ GGUFValueType.UINT64: np.uint64,+ GGUFValueType.INT64: np.int64,+ GGUFValueType.FLOAT64: np.float64,+ GGUFValueType.BOOL: np.bool_,+ }++ def __init__(self, path: os.PathLike[str] | str, mode: Literal['r', 'r+', 'c'] = 'r'):+ self.data = np.memmap(path, mode = mode)+ offs = 0++ # Check for GGUF magic+ if self._get(offs, np.uint32, override_order = '<')[0] != GGUF_MAGIC:+ raise ValueError('GGUF magic invalid')+ offs += 4++ # Check GGUF version+ temp_version = self._get(offs, np.uint32)+ if temp_version[0] & 65535 == 0:+ # If we get 0 here that means it's (probably) a GGUF file created for+ # the opposite byte order of the machine this script is running on.+ self.byte_order = 'S'+ temp_version = temp_version.view(temp_version.dtype.newbyteorder(self.byte_order))+ version = temp_version[0]+ if version not in READER_SUPPORTED_VERSIONS:+ raise ValueError(f'Sorry, file appears to be version {version} which we cannot handle')+ if sys.byteorder == "little":+ # Host is little endian+ host_endian = GGUFEndian.LITTLE+ swapped_endian = GGUFEndian.BIG+ else:+ # Sorry PDP or other weird systems that don't use BE or LE.+ host_endian = GGUFEndian.BIG+ swapped_endian = GGUFEndian.LITTLE+ self.endianess = swapped_endian if self.byte_order == "S" else host_endian+ self.fields: OrderedDict[str, ReaderField] = OrderedDict()+ self.tensors: list[ReaderTensor] = []+ offs += self._push_field(ReaderField(offs, 'GGUF.version', [temp_version], [0], [GGUFValueType.UINT32]))++ # Check tensor count and kv count+ temp_counts = self._get(offs, np.uint64, 2)+ offs += self._push_field(ReaderField(offs, 'GGUF.tensor_count', [temp_counts[:1]], [0], [GGUFValueType.UINT64]))+ offs += self._push_field(ReaderField(offs, 'GGUF.kv_count', [temp_counts[1:]], [0], [GGUFValueType.UINT64]))+ tensor_count, kv_count = temp_counts+ if tensor_count > GGUF_MAX_ARRAY_ELEMENTS:+ raise ValueError(f'Tensor count {tensor_count} exceeds maximum {GGUF_MAX_ARRAY_ELEMENTS}')+ if kv_count > GGUF_MAX_ARRAY_ELEMENTS:+ raise ValueError(f'KV count {kv_count} exceeds maximum {GGUF_MAX_ARRAY_ELEMENTS}')+ offs = self._build_fields(offs, kv_count)++ # Build Tensor Info Fields+ offs, tensors_fields = self._build_tensor_info(offs, tensor_count)+ new_align = self.fields.get('general.alignment')+ if new_align is not None:+ if new_align.types != [GGUFValueType.UINT32]:+ raise ValueError('Bad type for general.alignment field')+ self.alignment = new_align.parts[-1][0]+ # Ensure alignment is a non-zero power of two+ if self.alignment == 0 or (self.alignment & (self.alignment - 1)) != 0:+ raise ValueError('Invalid alignment: must be a non-zero power of two')+ padding = offs % self.alignment+ if padding != 0:+ offs += self.alignment - padding+ self.data_offset = offs+ self._build_tensors(offs, tensors_fields)++ _DT = TypeVar('_DT', bound = npt.DTypeLike)++ # Fetch a key/value metadata field by key.+ def get_field(self, key: str) -> Union[ReaderField, None]:+ return self.fields.get(key, None)++ # Fetch a tensor from the list by index.+ def get_tensor(self, idx: int) -> ReaderTensor:+ return self.tensors[idx]++ def _get(+ self, offset: int, dtype: npt.DTypeLike, count: int = 1, override_order: None | Literal['I', 'S', '<'] = None,+ ) -> npt.NDArray[Any]:+ count = int(count)+ itemsize = int(np.empty([], dtype = dtype).itemsize)+ end_offs = offset + itemsize * count+ arr = self.data[offset:end_offs].view(dtype=dtype)[:count]+ return arr.view(arr.dtype.newbyteorder(self.byte_order if override_order is None else override_order))++ def _push_field(self, field: ReaderField, skip_sum: bool = False) -> int:+ if field.name in self.fields:+ # TODO: add option to make this a warning and accept duplicate keys like below+ raise KeyError(f'Duplicate {field.name} already in list at offset {field.offset}')++ # logger.warning(f'Duplicate key {field.name} at offset {field.offset}')+ # self.fields[field.name + '_{}'.format(field.offset)] = field+ else:+ self.fields[field.name] = field+ return 0 if skip_sum else sum(int(part.nbytes) for part in field.parts)++ def _get_str(self, offset: int) -> tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint8]]:+ slen = self._get(offset, np.uint64)+ if int(slen[0]) > GGUF_MAX_STRING_LENGTH:+ raise ValueError(f'String length {int(slen[0])} exceeds maximum {GGUF_MAX_STRING_LENGTH}')+ if offset + 8 + int(slen[0]) > self.data.nbytes:+ raise ValueError(f'String length {int(slen[0])} exceeds remaining file size {self.data.nbytes - offset - 8}')+ return slen, self._get(offset + 8, np.uint8, slen[0])++ def _get_field_parts(+ self, orig_offs: int, raw_type: int,+ ) -> tuple[int, list[npt.NDArray[Any]], list[int], list[GGUFValueType]]:+ offs = orig_offs+ types: list[GGUFValueType] = []+ gtype = GGUFValueType(raw_type)+ types.append(gtype)+ # Handle strings.+ if gtype == GGUFValueType.STRING:+ sparts: list[npt.NDArray[Any]] = list(self._get_str(offs))+ size = sum(int(part.nbytes) for part in sparts)+ return size, sparts, [1], types+ # Check if it's a simple scalar type.+ nptype = self.gguf_scalar_to_np.get(gtype)+ if nptype is not None:+ val = self._get(offs, nptype)+ return int(val.nbytes), [val], [0], types+ # Handle arrays.+ if gtype == GGUFValueType.ARRAY:+ raw_itype = self._get(offs, np.uint32)+ offs += int(raw_itype.nbytes)+ alen = self._get(offs, np.uint64)+ if int(alen[0]) > GGUF_MAX_ARRAY_ELEMENTS:+ raise ValueError(f'Array length {int(alen[0])} exceeds maximum {GGUF_MAX_ARRAY_ELEMENTS}')+ offs += int(alen.nbytes)+ aparts: list[npt.NDArray[Any]] = [raw_itype, alen]+ data_idxs: list[int] = []+ # FIXME: Handle multi-dimensional arrays properly instead of flattening+ for idx in range(alen[0]):+ curr_size, curr_parts, curr_idxs, curr_types = self._get_field_parts(offs, raw_itype[0])+ if idx == 0:+ types += curr_types+ idxs_offs = len(aparts)+ aparts += curr_parts+ data_idxs += (idx + idxs_offs for idx in curr_idxs)+ offs += curr_size+ return offs - orig_offs, aparts, data_idxs, types+ # We can't deal with this one.+ raise ValueError(f'Unknown/unhandled field type {gtype}')++ def _get_tensor_info_field(self, orig_offs: int) -> ReaderField:+ offs = orig_offs++ # Get Tensor Name+ name_len, name_data = self._get_str(offs)+ offs += int(name_len.nbytes + name_data.nbytes)++ # Get Tensor Dimensions Count+ n_dims = self._get(offs, np.uint32)+ offs += int(n_dims.nbytes)+ if n_dims[0] > GGML_MAX_DIMS:+ raise ValueError(f'Tensor dimensions count {n_dims[0]} exceeds GGML_MAX_DIMS ({GGML_MAX_DIMS})')++ # Get Tensor Dimension Array+ dims = self._get(offs, np.uint64, n_dims[0])+ offs += int(dims.nbytes)++ # Get Tensor Encoding Scheme Type+ raw_dtype = self._get(offs, np.uint32)+ offs += int(raw_dtype.nbytes)++ # Get Tensor Offset+ offset_tensor = self._get(offs, np.uint64)+ offs += int(offset_tensor.nbytes)++ return ReaderField(+ orig_offs,+ str(bytes(name_data), encoding = 'utf-8'),+ [name_len, name_data, n_dims, dims, raw_dtype, offset_tensor],+ [1, 3, 4, 5],+ )++ def _build_fields(self, offs: int, count: int) -> int:+ for _ in range(count):+ orig_offs = offs+ kv_klen, kv_kdata = self._get_str(offs)+ offs += int(kv_klen.nbytes + kv_kdata.nbytes)+ raw_kv_type = self._get(offs, np.uint32)+ offs += int(raw_kv_type.nbytes)+ parts: list[npt.NDArray[Any]] = [kv_klen, kv_kdata, raw_kv_type]+ idxs_offs = len(parts)+ field_size, field_parts, field_idxs, field_types = self._get_field_parts(offs, raw_kv_type[0])+ parts += field_parts+ self._push_field(ReaderField(+ orig_offs,+ str(bytes(kv_kdata), encoding = 'utf-8'),+ parts,+ [idx + idxs_offs for idx in field_idxs],+ field_types,+ ), skip_sum = True)+ offs += field_size+ return offs++ def _build_tensor_info(self, offs: int, count: int) -> tuple[int, list[ReaderField]]:+ tensor_fields = []+ for _ in range(count):+ field = self._get_tensor_info_field(offs)+ offs += sum(int(part.nbytes) for part in field.parts)+ tensor_fields.append(field)+ return offs, tensor_fields++ def _build_tensors(self, start_offs: int, fields: list[ReaderField]) -> None:+ tensors = []+ tensor_names = set() # keep track of name to prevent duplicated tensors+ for field in fields:+ _name_len, name_data, _n_dims, dims, raw_dtype, offset_tensor = field.parts+ # check if there's any tensor having same name already in the list+ tensor_name = str(bytes(name_data), encoding = 'utf-8')+ if tensor_name in tensor_names:+ raise ValueError(f'Found duplicated tensor with name {tensor_name}')+ tensor_names.add(tensor_name)+ ggml_type = GGMLQuantizationType(raw_dtype[0])+ # use Python ints: np.prod on uint64 wraps silently on overflow+ n_elems = 1+ for dim in dims.tolist():+ n_elems *= int(dim)+ np_dims = tuple(reversed(dims.tolist()))+ block_size, type_size = GGML_QUANT_SIZES[ggml_type]+ n_bytes = n_elems * type_size // block_size+ data_offs = int(start_offs + offset_tensor[0])+ item_type: npt.DTypeLike+ if ggml_type == GGMLQuantizationType.F16:+ item_count = n_elems+ item_type = np.float16+ elif ggml_type == GGMLQuantizationType.F32:+ item_count = n_elems+ item_type = np.float32+ elif ggml_type == GGMLQuantizationType.F64:+ item_count = n_elems+ item_type = np.float64+ elif ggml_type == GGMLQuantizationType.I8:+ item_count = n_elems+ item_type = np.int8+ elif ggml_type == GGMLQuantizationType.I16:+ item_count = n_elems+ item_type = np.int16+ elif ggml_type == GGMLQuantizationType.I32:+ item_count = n_elems+ item_type = np.int32+ elif ggml_type == GGMLQuantizationType.I64:+ item_count = n_elems+ item_type = np.int64+ else:+ item_count = n_bytes+ item_type = np.uint8+ np_dims = quant_shape_to_byte_shape(np_dims, ggml_type)+ tensors.append(ReaderTensor(+ name = tensor_name,+ tensor_type = ggml_type,+ shape = dims,+ n_elements = n_elems,+ n_bytes = n_bytes,+ data_offset = data_offs,+ data = self._get(data_offs, item_type, item_count).reshape(np_dims),+ field = field,+ ))+ self.tensors = tensorsdiff --git a/python/freetoken/third_party/gguf/lazy.py b/python/freetoken/third_party/gguf/lazy.pynew file mode 100644index 00000000..a39f2232--- /dev/null+++ b/python/freetoken/third_party/gguf/lazy.py@@ -0,0 +1,293 @@+from __future__ import annotations+from abc import ABC, ABCMeta, abstractmethod++import logging+from typing import Any, Callable++import numpy as np+from numpy.typing import DTypeLike+++logger = logging.getLogger(__name__)+++class LazyMeta(ABCMeta):++ def __new__(cls, name: str, bases: tuple[type, ...], namespace: dict[str, Any], **kwargs):+ def __getattr__(self, name: str) -> Any:+ meta_attr = getattr(self._meta, name)+ if callable(meta_attr):+ return type(self)._wrap_fn(+ (lambda s, *args, **kwargs: getattr(s, name)(*args, **kwargs)),+ use_self=self,+ )+ elif isinstance(meta_attr, self._tensor_type):+ # e.g. self.T with torch.Tensor should still be wrapped+ return type(self)._wrap_fn(lambda s: getattr(s, name))(self)+ else:+ # no need to wrap non-tensor properties,+ # and they likely don't depend on the actual contents of the tensor+ return meta_attr++ namespace["__getattr__"] = __getattr__++ # need to make a builder for the wrapped wrapper to copy the name,+ # or else it fails with very cryptic error messages,+ # because somehow the same string would end up in every closures+ def mk_wrap(op_name: str, *, meta_noop: bool = False):+ # need to wrap the wrapper to get self+ def wrapped_special_op(self, *args, **kwargs):+ return type(self)._wrap_fn(+ getattr(type(self)._tensor_type, op_name),+ meta_noop=meta_noop,+ )(self, *args, **kwargs)+ return wrapped_special_op++ # special methods bypass __getattr__, so they need to be added manually+ # ref: https://docs.python.org/3/reference/datamodel.html#special-lookup+ # NOTE: doing this from a metaclass is very convenient+ # TODO: make this even more comprehensive+ for binary_op in (+ "lt", "le", "eq", "ne", "ge", "gt",+ "add", "and", "floordiv", "lshift", "mod", "mul", "matmul",+ "or", "pow", "rshift", "sub", "truediv", "xor",+ "iadd", "iand", "ifloordiv", "ilshift", "imod", "imul", "ior", "irshift", "isub", "ixor",+ "radd", "rand", "rfloordiv", "rmul", "ror", "rpow", "rsub", "rtruediv", "rxor",+ ):+ attr_name = f"__{binary_op}__"+ # evaluation on the meta tensor is needed in case there's broadcasting+ namespace[attr_name] = mk_wrap(attr_name, meta_noop=False)++ for unary_op in ("not", "abs", "invert", "neg", "pos"):+ attr_name = f"__{unary_op}__"+ # the result of these operators usually has the same shape and dtype as the input,+ # so evaluation on the meta tensor can be skipped.+ namespace[attr_name] = mk_wrap(attr_name, meta_noop=True)++ for special_op in (+ "getitem", "setitem", "len",+ ):+ attr_name = f"__{special_op}__"+ namespace[attr_name] = mk_wrap(attr_name, meta_noop=False)++ return super().__new__(cls, name, bases, namespace, **kwargs)+++# Tree of lazy tensors+class LazyBase(ABC, metaclass=LazyMeta):+ _tensor_type: type+ _meta: Any+ _data: Any | None+ _args: tuple+ _kwargs: dict[str, Any]+ _func: Callable[[Any], Any] | None++ def __init__(self, *, meta: Any, data: Any | None = None, args: tuple = (), kwargs: dict[str, Any] | None = None, func: Callable[[Any], Any] | None = None):+ super().__init__()+ self._meta = meta+ self._data = data+ self._args = args+ self._kwargs = kwargs if kwargs is not None else {}+ self._func = func+ assert self._func is not None or self._data is not None++ def __init_subclass__(cls) -> None:+ if "_tensor_type" not in cls.__dict__:+ raise TypeError(f"property '_tensor_type' must be defined for {cls!r}")+ return super().__init_subclass__()++ @staticmethod+ def _recurse_apply(o: Any, fn: Callable[[Any], Any]) -> Any:+ # TODO: dict and set+ if isinstance(o, (list, tuple)):+ L = []+ for item in o:+ L.append(LazyBase._recurse_apply(item, fn))+ if isinstance(o, tuple):+ L = tuple(L)+ return L+ elif isinstance(o, LazyBase):+ return fn(o)+ else:+ return o++ @classmethod+ def _wrap_fn(cls, fn: Callable, *, use_self: LazyBase | None = None, meta_noop: bool | DTypeLike | tuple[DTypeLike, Callable[[tuple[int, ...]], tuple[int, ...]]] = False) -> Callable[[Any], Any]:+ def wrapped_fn(*args, **kwargs):+ if kwargs is None:+ kwargs = {}+ args = ((use_self,) if use_self is not None else ()) + args++ meta_args = LazyBase._recurse_apply(args, lambda t: t._meta)+ # TODO: maybe handle tensors in kwargs too++ if isinstance(meta_noop, bool) and not meta_noop:+ try:+ res = fn(*meta_args, **kwargs)+ except NotImplementedError:+ # running some operations on PyTorch's Meta tensors can cause this exception+ res = None+ else:+ # some operators don't need to actually run on the meta tensors+ assert len(args) > 0+ res = args[0]+ assert isinstance(res, cls)+ res = res._meta+ # allow operations to override the dtype and shape+ if meta_noop is not True:+ if isinstance(meta_noop, tuple):+ dtype, shape = meta_noop+ assert callable(shape)+ res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape)) # ty: ignore[call-top-callable]+ else:+ res = cls.meta_with_dtype_and_shape(meta_noop, res.shape)++ if isinstance(res, cls._tensor_type):+ return cls(meta=cls.eager_to_meta(res), args=args, kwargs=kwargs, func=fn)+ elif isinstance(res, tuple) and all(isinstance(t, cls._tensor_type) for t in res):+ # share the evaluation between lazy tuple elements+ shared_args: list = [args, None]++ def eager_tuple_element(a: list[Any], i: int = 0, /, **kw) -> LazyBase:+ assert len(a) == 2+ if a[1] is None:+ a[1] = fn(*a[0], **kw)+ return a[1][i]+ return tuple(cls(meta=cls.eager_to_meta(res[i]), args=(shared_args, i), kwargs=kwargs, func=eager_tuple_element) for i in range(len(res)))+ else:+ del res # not needed+ # non-tensor return likely relies on the contents of the args+ # (e.g. the result of torch.equal)+ eager_args = cls.to_eager(args)+ return fn(*eager_args, **kwargs)+ return wrapped_fn++ @classmethod+ def to_eager(cls, t: Any) -> Any:+ def simple_to_eager(_t: LazyBase) -> Any:+ if _t._data is not None:+ return _t._data++ # NOTE: there's a recursion limit in Python (usually 1000)++ assert _t._func is not None+ _t._args = cls._recurse_apply(_t._args, simple_to_eager)+ _t._data = _t._func(*_t._args, **_t._kwargs)+ # sanity check+ assert _t._data is not None+ assert _t._data.dtype == _t._meta.dtype+ assert _t._data.shape == _t._meta.shape++ return _t._data++ # recurse into lists and/or tuples, keeping their structure+ return cls._recurse_apply(t, simple_to_eager)++ @classmethod+ def eager_to_meta(cls, t: Any) -> Any:+ return cls.meta_with_dtype_and_shape(t.dtype, t.shape)++ # must be overridden, meta tensor init is backend-specific+ @classmethod+ @abstractmethod+ def meta_with_dtype_and_shape(cls, dtype: Any, shape: Any) -> Any: pass++ @classmethod+ def from_eager(cls, t: Any) -> Any:+ if type(t) is cls:+ # already lazy+ return t+ elif isinstance(t, cls._tensor_type):+ return cls(meta=cls.eager_to_meta(t), data=t)+ else:+ return TypeError(f"{type(t)!r} is not compatible with {cls._tensor_type!r}")+++class LazyNumpyTensor(LazyBase):+ _tensor_type = np.ndarray++ shape: tuple[int, ...] # Makes the type checker happy in quants.py++ @classmethod+ def meta_with_dtype_and_shape(cls, dtype: DTypeLike, shape: tuple[int, ...]) -> np.ndarray[Any, Any]:+ # The initial idea was to use np.nan as the fill value,+ # but non-float types like np.int16 can't use that.+ # So zero it is.+ cheat = np.zeros(1, dtype)+ return np.lib.stride_tricks.as_strided(cheat, shape, (0 for _ in shape))++ def astype(self, dtype, *args, **kwargs):+ meta = type(self).meta_with_dtype_and_shape(dtype, self._meta.shape)+ full_args = (self, dtype,) + args+ return type(self)(meta=meta, args=full_args, kwargs=kwargs, func=(lambda a, *args, **kwargs: a.astype(*args, **kwargs)))++ def tofile(self, *args, **kwargs):+ eager = LazyNumpyTensor.to_eager(self)+ return eager.tofile(*args, **kwargs)++ # TODO: __array_function__+++# Tensor written to file one row-chunk at a time+class LazyChunkedTensor:++ def __init__(+ self, chunks: list[Callable[[], np.ndarray]], shape: tuple[int, ...], dtype: DTypeLike,+ qtype: Any = None, byteswap: bool = False,+ ):+ self._chunks = chunks+ self._qtype = qtype+ self._byteswap = byteswap+ self.shape = tuple(shape)+ self.dtype = np.dtype(dtype)++ @property+ def nbytes(self) -> int:+ n = self.dtype.itemsize+ for d in self.shape:+ n *= d+ return n++ def numpy(self) -> LazyChunkedTensor:+ return self++ def __array__(self, *args, **kwargs):+ # numpy would otherwise make a 1-element object array of self, and write 8 bytes+ raise TypeError("LazyChunkedTensor cannot become an ndarray, it is written in chunks")++ def quantize(self, qtype: Any) -> LazyChunkedTensor:+ from .constants import GGMLQuantizationType+ from .quants import QuantError, quant_shape_to_byte_shape++ if qtype == GGMLQuantizationType.F32:+ shape, dtype = self.shape, np.dtype(np.float32)+ elif qtype == GGMLQuantizationType.F16:+ shape, dtype = self.shape, np.dtype(np.float16)+ else:+ try:+ shape, dtype = quant_shape_to_byte_shape(self.shape, qtype), np.dtype(np.uint8)+ except ValueError as e:+ # raised here and not per chunk, so callers can still fall back to F16+ raise QuantError(str(e)) from e+ return LazyChunkedTensor(self._chunks, shape, dtype, qtype, self._byteswap)++ def byteswap(self, inplace: bool = False) -> LazyChunkedTensor:+ if inplace:+ raise NotImplementedError("a chunked tensor cannot be byteswapped in place")+ return LazyChunkedTensor(self._chunks, self.shape, self.dtype, self._qtype, not self._byteswap)++ def tofile(self, *args, **kwargs) -> None:+ from .quants import quantize++ written = 0+ for load_chunk in self._chunks:+ chunk = load_chunk()+ if self._qtype is not None:+ # exact only because chunks split on rows, and blocks never cross one+ chunk = quantize(chunk, self._qtype)+ if self._byteswap:+ chunk = chunk.byteswap(inplace=False)+ chunk.tofile(*args, **kwargs)+ written += chunk.nbytes+ del chunk+ assert written == self.nbytes, f"chunked tensor wrote {written} bytes, expected {self.nbytes}"diff --git a/python/freetoken/third_party/gguf/quants.py b/python/freetoken/third_party/gguf/quants.pynew file mode 100644index 00000000..80966b6e--- /dev/null+++ b/python/freetoken/third_party/gguf/quants.py@@ -0,0 +1,1378 @@+from __future__ import annotations+from abc import ABC, abstractmethod+from typing import Any, Callable, Sequence+from math import log2, ceil++from numpy.typing import DTypeLike++from .constants import GGML_QUANT_SIZES, GGMLQuantizationType, QK_K+from .lazy import LazyNumpyTensor++import numpy as np+++def quant_shape_to_byte_shape(shape: Sequence[int], quant_type: GGMLQuantizationType) -> tuple[int, ...]:+ block_size, type_size = GGML_QUANT_SIZES[quant_type]+ if shape[-1] % block_size != 0:+ raise ValueError(f"Quantized tensor row size ({shape[-1]}) is not a multiple of {quant_type.name} block size ({block_size})")+ return (*shape[:-1], shape[-1] // block_size * type_size)+++def quant_shape_from_byte_shape(shape: Sequence[int], quant_type: GGMLQuantizationType) -> tuple[int, ...]:+ block_size, type_size = GGML_QUANT_SIZES[quant_type]+ if shape[-1] % type_size != 0:+ raise ValueError(f"Quantized tensor bytes per row ({shape[-1]}) is not a multiple of {quant_type.name} type size ({type_size})")+ return (*shape[:-1], shape[-1] // type_size * block_size)+++# This is faster than np.vectorize and np.apply_along_axis because it works on more than one row at a time+def _apply_over_grouped_rows(func: Callable[[np.ndarray], np.ndarray], arr: np.ndarray, otype: DTypeLike, oshape: tuple[int, ...]) -> np.ndarray:+ rows = arr.reshape((-1, arr.shape[-1]))+ assert len(rows.shape)+ osize = 1+ for dim in oshape:+ osize *= dim+ out = np.empty(shape=osize, dtype=otype)+ # compute over groups of 16 rows (arbitrary, but seems good for performance)+ n_groups = (rows.shape[0] // 16) or 1+ np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out)+ return out.reshape(oshape)+++# round away from zero+# ref: https://stackoverflow.com/a/59143326/22827863+def np_roundf(n: np.ndarray) -> np.ndarray:+ a = abs(n)+ floored = np.floor(a)+ b = floored + np.floor(2 * (a - floored))+ return np.sign(n) * b+++class QuantError(Exception): ...+++_type_traits: dict[GGMLQuantizationType, type[__Quant]] = {}+++def quantize(data: np.ndarray, qtype: GGMLQuantizationType) -> np.ndarray:+ if qtype == GGMLQuantizationType.F32:+ return data.astype(np.float32, copy=False)+ elif qtype == GGMLQuantizationType.F16:+ return data.astype(np.float16, copy=False)+ elif (q := _type_traits.get(qtype)) is not None:+ return q.quantize(data)+ else:+ raise NotImplementedError(f"Quantization for {qtype.name} is not yet implemented")+++def dequantize(data: np.ndarray, qtype: GGMLQuantizationType) -> np.ndarray:+ if qtype == GGMLQuantizationType.F32:+ return data.view(np.float32)+ elif qtype == GGMLQuantizationType.F16:+ return data.view(np.float16).astype(np.float32)+ elif (q := _type_traits.get(qtype)) is not None:+ return q.dequantize(data)+ else:+ raise NotImplementedError(f"Dequantization for {qtype.name} is not yet implemented")+++class __Quant(ABC):+ qtype: GGMLQuantizationType+ block_size: int+ type_size: int++ grid: np.ndarray[Any, np.dtype[np.float32]] | None = None+ grid_shape: tuple[int, int] = (0, 0)+ grid_map: tuple[int | float, ...] = ()+ grid_hex: bytes | None = None++ def __init__(self):+ return TypeError("Quant conversion classes can't have instances")++ def __init_subclass__(cls, qtype: GGMLQuantizationType) -> None:+ cls.qtype = qtype+ cls.block_size, cls.type_size = GGML_QUANT_SIZES[qtype]+ cls.__quantize_lazy: Any = LazyNumpyTensor._wrap_fn(+ cls.__quantize_array,+ meta_noop=(np.uint8, cls.__shape_to_bytes)+ )+ cls.__dequantize_lazy: Any = LazyNumpyTensor._wrap_fn(+ cls.__dequantize_array,+ meta_noop=(np.float32, cls.__shape_from_bytes)+ )+ assert qtype not in _type_traits+ _type_traits[qtype] = cls++ @classmethod+ def init_grid(cls):+ if cls.grid is not None or cls.grid_hex is None:+ return++ bits_per_elem = ceil(log2(len(cls.grid_map)))+ assert bits_per_elem != 0, cls.qtype.name+ elems_per_byte = 8 // bits_per_elem++ grid = np.frombuffer(cls.grid_hex, dtype=np.uint8)+ # decode hexadecimal chars from grid+ grid = grid.reshape((-1, 2))+ grid = (np.where(grid > 0x40, grid + 9, grid) & 0x0F) << np.array([4, 0], dtype=np.uint8).reshape((1, 2))+ grid = grid[..., 0] | grid[..., 1]+ # unpack the grid values+ grid = grid.reshape((-1, 1)) >> np.array([i for i in range(0, 8, 8 // elems_per_byte)], dtype=np.uint8).reshape((1, elems_per_byte))+ grid = (grid & ((1 << bits_per_elem) - 1)).reshape((-1, 1))+ grid_map = np.array(cls.grid_map, dtype=np.float32).reshape((1, -1))+ grid = np.take_along_axis(grid_map, grid, axis=-1)+ cls.grid = grid.reshape((1, 1, *cls.grid_shape))++ @classmethod+ @abstractmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ raise NotImplementedError++ @classmethod+ @abstractmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ raise NotImplementedError++ @classmethod+ def quantize_rows(cls, rows: np.ndarray) -> np.ndarray:+ rows = rows.astype(np.float32, copy=False)+ shape = rows.shape+ n_blocks = rows.size // cls.block_size+ blocks = rows.reshape((n_blocks, cls.block_size))+ blocks = cls.quantize_blocks(blocks)+ assert blocks.dtype == np.uint8+ assert blocks.shape[-1] == cls.type_size+ return blocks.reshape(cls.__shape_to_bytes(shape))++ @classmethod+ def dequantize_rows(cls, rows: np.ndarray) -> np.ndarray:+ rows = rows.view(np.uint8)+ shape = rows.shape+ n_blocks = rows.size // cls.type_size+ blocks = rows.reshape((n_blocks, cls.type_size))+ blocks = cls.dequantize_blocks(blocks)+ assert blocks.dtype == np.float32+ assert blocks.shape[-1] == cls.block_size+ return blocks.reshape(cls.__shape_from_bytes(shape))++ @classmethod+ def __shape_to_bytes(cls, shape: Sequence[int]):+ return quant_shape_to_byte_shape(shape, cls.qtype)++ @classmethod+ def __shape_from_bytes(cls, shape: Sequence[int]):+ return quant_shape_from_byte_shape(shape, cls.qtype)++ @classmethod+ def __quantize_array(cls, array: np.ndarray) -> np.ndarray:+ return _apply_over_grouped_rows(cls.quantize_rows, arr=array, otype=np.uint8, oshape=cls.__shape_to_bytes(array.shape))++ @classmethod+ def __dequantize_array(cls, array: np.ndarray) -> np.ndarray:+ cls.init_grid()+ return _apply_over_grouped_rows(cls.dequantize_rows, arr=array, otype=np.float32, oshape=cls.__shape_from_bytes(array.shape))++ @classmethod+ def __quantize_lazy(cls, lazy_tensor: LazyNumpyTensor, /) -> Any:+ pass++ @classmethod+ def __dequantize_lazy(cls, lazy_tensor: LazyNumpyTensor, /) -> Any:+ pass++ @classmethod+ def can_quantize(cls, tensor: np.ndarray | LazyNumpyTensor) -> bool:+ return tensor.shape[-1] % cls.block_size == 0++ @classmethod+ def quantize(cls, tensor: np.ndarray | LazyNumpyTensor) -> np.ndarray:+ if not cls.can_quantize(tensor):+ raise QuantError(f"Can't quantize tensor with shape {tensor.shape} to {cls.qtype.name}")+ if isinstance(tensor, LazyNumpyTensor):+ return cls.__quantize_lazy(tensor)+ else:+ return cls.__quantize_array(tensor)++ @classmethod+ def dequantize(cls, tensor: np.ndarray | LazyNumpyTensor) -> np.ndarray:+ if isinstance(tensor, LazyNumpyTensor):+ return cls.__dequantize_lazy(tensor)+ else:+ return cls.__dequantize_array(tensor)+++class BF16(__Quant, qtype=GGMLQuantizationType.BF16):+ @classmethod+ # same as ggml_compute_fp32_to_bf16 in ggml-impl.h+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n = blocks.view(np.uint32)+ # force nan to quiet+ n = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n)+ # round to nearest even+ n = (np.uint64(n) + (0x7fff + ((n >> 16) & 1))) >> 16+ return n.astype(np.uint16).view(np.uint8)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ return (blocks.view(np.int16).astype(np.int32) << 16).view(np.float32)+++class Q4_0(__Quant, qtype=GGMLQuantizationType.Q4_0):+ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ imax = abs(blocks).argmax(axis=-1, keepdims=True)+ max = np.take_along_axis(blocks, imax, axis=-1)++ d = max / -8+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ qs = np.trunc((blocks * id) + np.float32(8.5), dtype=np.float32).astype(np.uint8).clip(0, 15)++ qs = qs.reshape((n_blocks, 2, cls.block_size // 2))+ qs = qs[..., 0, :] | (qs[..., 1, :] << np.uint8(4))++ d = d.astype(np.float16).view(np.uint8)++ return np.concatenate([d, qs], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, qs = np.hsplit(blocks, [2])++ d = d.view(np.float16).astype(np.float32)++ qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))+ qs = (qs & np.uint8(0x0F)).reshape((n_blocks, -1)).astype(np.int8) - np.int8(8)++ return (d * qs.astype(np.float32))+++class Q4_1(__Quant, qtype=GGMLQuantizationType.Q4_1):+ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ max = blocks.max(axis=-1, keepdims=True)+ min = blocks.min(axis=-1, keepdims=True)++ d = (max - min) / 15+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ qs = np.trunc((blocks - min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 15)++ qs = qs.reshape((n_blocks, 2, cls.block_size // 2))+ qs = qs[..., 0, :] | (qs[..., 1, :] << np.uint8(4))++ d = d.astype(np.float16).view(np.uint8)+ m = min.astype(np.float16).view(np.uint8)++ return np.concatenate([d, m, qs], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ m, qs = np.hsplit(rest, [2])++ d = d.view(np.float16).astype(np.float32)+ m = m.view(np.float16).astype(np.float32)++ qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))+ qs = (qs & np.uint8(0x0F)).reshape((n_blocks, -1)).astype(np.float32)++ return (d * qs) + m+++class Q5_0(__Quant, qtype=GGMLQuantizationType.Q5_0):+ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ imax = abs(blocks).argmax(axis=-1, keepdims=True)+ max = np.take_along_axis(blocks, imax, axis=-1)++ d = max / -16+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ q = np.trunc((blocks * id) + np.float32(16.5), dtype=np.float32).astype(np.uint8).clip(0, 31)++ qs = q.reshape((n_blocks, 2, cls.block_size // 2))+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))++ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)++ d = d.astype(np.float16).view(np.uint8)++ return np.concatenate([d, qh, qs], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ qh, qs = np.hsplit(rest, [4])++ d = d.view(np.float16).astype(np.float32)+ qh = qh.view(np.uint32)++ qh = qh.reshape((n_blocks, 1)) >> np.array([i for i in range(32)], dtype=np.uint32).reshape((1, 32))+ ql = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))+ qh = (qh & np.uint32(0x01)).astype(np.uint8)+ ql = (ql & np.uint8(0x0F)).reshape((n_blocks, -1))++ qs = (ql | (qh << np.uint8(4))).astype(np.int8) - np.int8(16)++ return (d * qs.astype(np.float32))+++class Q5_1(__Quant, qtype=GGMLQuantizationType.Q5_1):+ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ max = blocks.max(axis=-1, keepdims=True)+ min = blocks.min(axis=-1, keepdims=True)++ d = (max - min) / 31+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ q = np.trunc((blocks - min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 31)++ qs = q.reshape((n_blocks, 2, cls.block_size // 2))+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))++ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)++ d = d.astype(np.float16).view(np.uint8)+ m = min.astype(np.float16).view(np.uint8)++ return np.concatenate([d, m, qh, qs], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ m, rest = np.hsplit(rest, [2])+ qh, qs = np.hsplit(rest, [4])++ d = d.view(np.float16).astype(np.float32)+ m = m.view(np.float16).astype(np.float32)+ qh = qh.view(np.uint32)++ qh = qh.reshape((n_blocks, 1)) >> np.array([i for i in range(32)], dtype=np.uint32).reshape((1, 32))+ ql = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))+ qh = (qh & np.uint32(0x01)).astype(np.uint8)+ ql = (ql & np.uint8(0x0F)).reshape((n_blocks, -1))++ qs = (ql | (qh << np.uint8(4))).astype(np.float32)++ return (d * qs) + m+++class Q8_0(__Quant, qtype=GGMLQuantizationType.Q8_0):+ @classmethod+ # Implementation of Q8_0 with bit-exact same results as reference implementation in ggml-quants.c+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:++ d = abs(blocks).max(axis=1, keepdims=True) / 127+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ qs = np_roundf(blocks * id)++ # (n_blocks, 2)+ d = d.astype(np.float16).view(np.uint8)+ # (n_blocks, block_size)+ qs = qs.astype(np.int8).view(np.uint8)++ return np.concatenate([d, qs], axis=1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ d, x = np.split(blocks, [2], axis=1)+ d = d.view(np.float16).astype(np.float32)+ x = x.view(np.int8).astype(np.float32)++ return (x * d)+++class Q2_K(__Quant, qtype=GGMLQuantizationType.Q2_K):+ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ scales, rest = np.hsplit(blocks, [QK_K // 16])+ qs, rest = np.hsplit(rest, [QK_K // 4])+ d, dmin = np.hsplit(rest, [2])++ d = d.view(np.float16).astype(np.float32)+ dmin = dmin.view(np.float16).astype(np.float32)++ # (n_blocks, 16, 1)+ dl = (d * (scales & 0xF).astype(np.float32)).reshape((n_blocks, QK_K // 16, 1))+ ml = (dmin * (scales >> 4).astype(np.float32)).reshape((n_blocks, QK_K // 16, 1))++ shift = np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))++ qs = (qs.reshape((n_blocks, -1, 1, 32)) >> shift) & np.uint8(3)++ qs = qs.reshape((n_blocks, QK_K // 16, 16)).astype(np.float32)++ qs = dl * qs - ml++ return qs.reshape((n_blocks, -1))+++class Q3_K(__Quant, qtype=GGMLQuantizationType.Q3_K):+ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ hmask, rest = np.hsplit(blocks, [QK_K // 8])+ qs, rest = np.hsplit(rest, [QK_K // 4])+ scales, d = np.hsplit(rest, [12])++ d = d.view(np.float16).astype(np.float32)++ # The scales are packed at 6-bit each in this pattern:+ # 0: IIIIAAAA+ # 1: JJJJBBBB+ # 2: KKKKCCCC+ # 3: LLLLDDDD+ # 4: MMMMEEEE+ # 5: NNNNFFFF+ # 6: OOOOGGGG+ # 7: PPPPHHHH+ # 8: MMIIEEAA+ # 9: NNJJFFBB+ # 10: OOKKGGCC+ # 11: PPLLHHDD+ lscales, hscales = np.hsplit(scales, [8])+ lscales = lscales.reshape((n_blocks, 1, 8)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 2, 1))+ lscales = lscales.reshape((n_blocks, 16))+ hscales = hscales.reshape((n_blocks, 1, 4)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 4, 1))+ hscales = hscales.reshape((n_blocks, 16))+ scales = (lscales & np.uint8(0x0F)) | ((hscales & np.uint8(0x03)) << np.uint8(4))+ scales = (scales.astype(np.int8) - np.int8(32)).astype(np.float32)++ dl = (d * scales).reshape((n_blocks, 16, 1))++ ql = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))+ qh = hmask.reshape(n_blocks, -1, 1, 32) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8, 1))+ ql = ql.reshape((n_blocks, 16, QK_K // 16)) & np.uint8(3)+ qh = (qh.reshape((n_blocks, 16, QK_K // 16)) & np.uint8(1))+ qh = qh ^ np.uint8(1) # strangely, the offset is zero when the bitmask is 1+ q = (ql.astype(np.int8) - (qh << np.uint8(2)).astype(np.int8)).astype(np.float32)++ return (dl * q).reshape((n_blocks, QK_K))+++class Q4_K(__Quant, qtype=GGMLQuantizationType.Q4_K):+ K_SCALE_SIZE = 12++ @staticmethod+ def get_scale_min(scales: np.ndarray) -> tuple[np.ndarray, np.ndarray]:+ n_blocks = scales.shape[0]+ scales = scales.view(np.uint8)+ ### Unpacking the following: ###+ # 0 EEAAAAAA+ # 1 FFBBBBBB+ # 2 GGCCCCCC+ # 3 HHDDDDDD+ # 4 eeaaaaaa+ # 5 ffbbbbbb+ # 6 ggcccccc+ # 7 hhdddddd+ # 8 eeeeEEEE+ # 9 ffffFFFF+ # 10 ggggGGGG+ # 11 hhhhHHHH+ scales = scales.reshape((n_blocks, 3, 4))+ d, m, m_d = np.split(scales, 3, axis=-2)++ sc = np.concatenate([d & 0x3F, (m_d & 0x0F) | ((d >> 2) & 0x30)], axis=-1)+ min = np.concatenate([m & 0x3F, (m_d >> 4) | ((m >> 2) & 0x30)], axis=-1)++ return (sc.reshape((n_blocks, 8)), min.reshape((n_blocks, 8)))++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ dmin, rest = np.hsplit(rest, [2])+ scales, qs = np.hsplit(rest, [cls.K_SCALE_SIZE])++ d = d.view(np.float16).astype(np.float32)+ dmin = dmin.view(np.float16).astype(np.float32)++ sc, m = Q4_K.get_scale_min(scales)++ d = (d * sc.astype(np.float32)).reshape((n_blocks, -1, 1))+ dm = (dmin * m.astype(np.float32)).reshape((n_blocks, -1, 1))++ qs = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))+ qs = (qs & np.uint8(0x0F)).reshape((n_blocks, -1, 32)).astype(np.float32)++ return (d * qs - dm).reshape((n_blocks, QK_K))+++class Q5_K(__Quant, qtype=GGMLQuantizationType.Q5_K):+ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ dmin, rest = np.hsplit(rest, [2])+ scales, rest = np.hsplit(rest, [Q4_K.K_SCALE_SIZE])+ qh, qs = np.hsplit(rest, [QK_K // 8])++ d = d.view(np.float16).astype(np.float32)+ dmin = dmin.view(np.float16).astype(np.float32)++ sc, m = Q4_K.get_scale_min(scales)++ d = (d * sc.astype(np.float32)).reshape((n_blocks, -1, 1))+ dm = (dmin * m.astype(np.float32)).reshape((n_blocks, -1, 1))++ ql = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))+ qh = qh.reshape((n_blocks, -1, 1, 32)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8, 1))+ ql = (ql & np.uint8(0x0F)).reshape((n_blocks, -1, 32))+ qh = (qh & np.uint8(0x01)).reshape((n_blocks, -1, 32))+ q = (ql | (qh << np.uint8(4))).astype(np.float32)++ return (d * q - dm).reshape((n_blocks, QK_K))+++class Q6_K(__Quant, qtype=GGMLQuantizationType.Q6_K):+ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ ql, rest = np.hsplit(blocks, [QK_K // 2])+ qh, rest = np.hsplit(rest, [QK_K // 4])+ scales, d = np.hsplit(rest, [QK_K // 16])++ scales = scales.view(np.int8).astype(np.float32)+ d = d.view(np.float16).astype(np.float32)+ d = (d * scales).reshape((n_blocks, QK_K // 16, 1))++ ql = ql.reshape((n_blocks, -1, 1, 64)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))+ ql = (ql & np.uint8(0x0F)).reshape((n_blocks, -1, 32))+ qh = qh.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))+ qh = (qh & np.uint8(0x03)).reshape((n_blocks, -1, 32))+ q = (ql | (qh << np.uint8(4))).astype(np.int8) - np.int8(32)+ q = q.reshape((n_blocks, QK_K // 16, -1)).astype(np.float32)++ return (d * q).reshape((n_blocks, QK_K))+++class TQ1_0(__Quant, qtype=GGMLQuantizationType.TQ1_0):+ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d = abs(blocks).max(axis=-1, keepdims=True)+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ qs = np_roundf(blocks * id)+ qs = (qs.astype(np.int8) + np.int8(1)).astype(np.uint8)++ qs0, qs1, qh = qs[..., :(32 * 5)], qs[..., (32 * 5):(48 * 5)], qs[..., (48 * 5):]+ qs0 = qs0.reshape((n_blocks, -1, 5, 32)) * np.array([81, 27, 9, 3, 1], dtype=np.uint8).reshape((1, 1, 5, 1))+ qs0 = np.sum(qs0, axis=-2).reshape((n_blocks, -1))+ qs1 = qs1.reshape((n_blocks, -1, 5, 16)) * np.array([81, 27, 9, 3, 1], dtype=np.uint8).reshape((1, 1, 5, 1))+ qs1 = np.sum(qs1, axis=-2).reshape((n_blocks, -1))+ qh = qh.reshape((n_blocks, -1, 4, 4)) * np.array([81, 27, 9, 3], dtype=np.uint8).reshape((1, 1, 4, 1))+ qh = np.sum(qh, axis=-2).reshape((n_blocks, -1))+ qs = np.concatenate([qs0, qs1, qh], axis=-1)+ qs = (qs.astype(np.uint16) * 256 + (243 - 1)) // 243++ qs = qs.astype(np.uint8)+ d = d.astype(np.float16).view(np.uint8)++ return np.concatenate([qs, d], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ qs, rest = np.hsplit(blocks, [(QK_K - 4 * QK_K // 64) // 5])+ qh, d = np.hsplit(rest, [QK_K // 64])++ d = d.view(np.float16).astype(np.float32)++ qs0, qs1 = qs[..., :32], qs[..., 32:]+ qs0 = qs0.reshape((n_blocks, -1, 1, 32)) * np.array([1, 3, 9, 27, 81], dtype=np.uint8).reshape((1, 1, 5, 1))+ qs0 = qs0.reshape((n_blocks, -1))+ qs1 = qs1.reshape((n_blocks, -1, 1, 16)) * np.array([1, 3, 9, 27, 81], dtype=np.uint8).reshape((1, 1, 5, 1))+ qs1 = qs1.reshape((n_blocks, -1))+ qh = qh.reshape((n_blocks, -1, 1, 4)) * np.array([1, 3, 9, 27], dtype=np.uint8).reshape((1, 1, 4, 1))+ qh = qh.reshape((n_blocks, -1))+ qs = np.concatenate([qs0, qs1, qh], axis=-1)+ qs = ((qs.astype(np.uint16) * 3) >> 8).astype(np.int8) - np.int8(1)++ return (d * qs.astype(np.float32))+++class TQ2_0(__Quant, qtype=GGMLQuantizationType.TQ2_0):+ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d = abs(blocks).max(axis=-1, keepdims=True)+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ qs = np_roundf(blocks * id)+ qs = (qs.astype(np.int8) + np.int8(1)).astype(np.uint8)++ qs = qs.reshape((n_blocks, -1, 4, 32)) << np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))+ qs = qs[..., 0, :] | qs[..., 1, :] | qs[..., 2, :] | qs[..., 3, :]+ qs = qs.reshape((n_blocks, -1))++ d = d.astype(np.float16).view(np.uint8)++ return np.concatenate([qs, d], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ qs, d = np.hsplit(blocks, [QK_K // 4])++ d = d.view(np.float16).astype(np.float32)++ qs = qs.reshape((n_blocks, -1, 1, 32)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4, 1))+ qs = (qs & 0x03).reshape((n_blocks, -1)).astype(np.int8) - np.int8(1)++ return (d * qs.astype(np.float32))+++class MXFP4(__Quant, qtype=GGMLQuantizationType.MXFP4):+ # e2m1 values (doubled)+ # ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf+ kvalues = (0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12)++ @staticmethod+ # see ggml_e8m0_to_fp32_half in ggml-impl.h+ def e8m0_to_fp32_half(x: np.ndarray) -> np.ndarray:+ bits = np.where(x < 2, np.uint32(0x00200000) << np.uint32(x), np.uint32(x - 1) << np.uint32(23))+ return bits.view(np.float32)++ @classmethod+ def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d = abs(blocks).max(axis=-1, keepdims=True)++ with np.errstate(divide="ignore"):+ e = np.where(d > 0, np.floor(np.log2(d)) - 2 + 127, 0).astype(np.uint8)++ d = cls.e8m0_to_fp32_half(e)++ kvalues = np.array(cls.kvalues, dtype=np.int8).reshape((1, 1, 16))++ errs = np.abs(d.reshape((n_blocks, 1, 1)) * kvalues.astype(np.float32) - blocks.reshape((n_blocks, cls.block_size, 1)))+ best = np.argmin(errs, axis=-1, keepdims=True)++ qs = best.reshape(n_blocks, 2, cls.block_size // 2).astype(np.uint8)+ qs = qs[:, 0] | (qs[:, 1] << np.uint8(4))++ qs = qs.reshape((n_blocks, cls.block_size // 2))++ return np.concatenate([e, qs], axis=-1)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ e, qs = np.hsplit(blocks, [1])++ d = cls.e8m0_to_fp32_half(e)++ qs = qs.reshape((n_blocks, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 2, 1))+ qs = (qs & np.uint8(0x0F)).view(np.int8)++ kvalues = np.array(cls.kvalues, dtype=np.int8).reshape(1, 1, 16)+ qs = np.take_along_axis(kvalues, qs, axis=-1).reshape((n_blocks, cls.block_size))++ return (d * qs.astype(np.float32))+++class NVFP4(__Quant, qtype=GGMLQuantizationType.NVFP4):+ # E2M1 values doubled (kvalues_mxfp4 convention)+ kvalues = (0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12)++ @staticmethod+ def ue4m3_to_fp32(x: np.ndarray) -> np.ndarray:+ """Decode unsigned E4M3 (bias=7) to float, with 0.5 factor for kvalues convention."""+ exp = (x >> 3).astype(np.int32) & 0xF+ man = (x & 0x7).astype(np.float32)+ raw = np.where(+ exp == 0,+ man * 2**-9,+ (1.0 + man / 8.0) * (2.0 ** (exp.astype(np.float32) - 7)))+ return np.where((x == 0) | (x == 0x7F), 0.0, raw * 0.5)++ @staticmethod+ def fp32_to_ue4m3(x: np.ndarray) -> np.ndarray:+ """Vectorized float32 to unsigned E4M3, matching ggml_fp32_to_ue4m3 in C."""+ x = np.clip(x, 0.0, 448.0).astype(np.float32)+ bits = x.view(np.uint32)+ fp32_exp = ((bits >> 23) & 0xFF).astype(np.int32) - 127+ fp32_man = ((bits >> 20) & 0x7).astype(np.int32)+ ue4m3_exp = fp32_exp + 7++ # Subnormal+ sub_man = np.clip((x * 512.0 + 0.5).astype(np.int32), 0, 7)+ sub_result = np.where(sub_man >= 1, sub_man, 0).astype(np.uint8)++ # Normal with rounding+ round_bit = ((bits >> 19) & 1).astype(np.int32)+ man = fp32_man + round_bit+ exp = ue4m3_exp.copy()+ overflow = man > 7+ man = np.where(overflow, 0, man)+ exp = np.where(overflow, exp + 1, exp)+ normal_result = np.where(exp >= 15, np.uint8(0x7E), ((exp << 3) | man).astype(np.uint8))++ return np.where(x <= 0.0, np.uint8(0),+ np.where(ue4m3_exp <= 0, sub_result,+ np.where(ue4m3_exp >= 15, np.uint8(0x7E), normal_result)))++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_super = blocks.shape[0]++ d_bytes, qs = np.hsplit(blocks, [4])+ d = cls.ue4m3_to_fp32(d_bytes).reshape(n_super, 4, 1) # (n_super, 4, 1)++ qs = qs.reshape(n_super, 4, 8)+ lo = (qs & np.uint8(0x0F)).view(np.int8)+ hi = (qs >> np.uint8(4)).view(np.int8)+ vals = np.concatenate([lo, hi], axis=-1) # (n_super, 4, 16)++ kvalues = np.array(cls.kvalues, dtype=np.int8).reshape(1, 1, 16)+ vals = np.take_along_axis(kvalues, vals, axis=-1)++ return (d * vals.astype(np.float32)).reshape(n_super, 64)+++class IQ2_XXS(__Quant, qtype=GGMLQuantizationType.IQ2_XXS):+ ksigns: bytes = (+ b"\x00\x81\x82\x03\x84\x05\x06\x87\x88\x09\x0a\x8b\x0c\x8d\x8e\x0f"+ b"\x90\x11\x12\x93\x14\x95\x96\x17\x18\x99\x9a\x1b\x9c\x1d\x1e\x9f"+ b"\xa0\x21\x22\xa3\x24\xa5\xa6\x27\x28\xa9\xaa\x2b\xac\x2d\x2e\xaf"+ b"\x30\xb1\xb2\x33\xb4\x35\x36\xb7\xb8\x39\x3a\xbb\x3c\xbd\xbe\x3f"+ b"\xc0\x41\x42\xc3\x44\xc5\xc6\x47\x48\xc9\xca\x4b\xcc\x4d\x4e\xcf"+ b"\x50\xd1\xd2\x53\xd4\x55\x56\xd7\xd8\x59\x5a\xdb\x5c\xdd\xde\x5f"+ b"\x60\xe1\xe2\x63\xe4\x65\x66\xe7\xe8\x69\x6a\xeb\x6c\xed\xee\x6f"+ b"\xf0\x71\x72\xf3\x74\xf5\xf6\x77\x78\xf9\xfa\x7b\xfc\x7d\x7e\xff"+ )++ # iq2xxs_grid, but with each byte of the original packed in 2 bits,+ # by mapping 0x08 to 0, 0x19 to 1, and 0x2b to 2.+ grid_shape = (256, 8)+ grid_map = (0x08, 0x19, 0x2b)+ grid_hex = (+ b"00000200050008000a00110014002000220028002a0041004400500058006100"+ b"6400800082008a00a20001010401100115014001840198010002020222028202"+ b"010404041004210424044004420448046004810484049004a404000502050805"+ b"200546056905800591050906100640068406a406000805080808140828084108"+ b"440850085208880804094009020a140a01100410101021104010601084109010"+ b"951000110811201150115a118011241245120014081420142514491480141815"+ b"6215001616160118041810184018811800190519a019511a002002200a204420"+ b"6120802082202921482100220222012404241024402456240025412564259026"+ b"082820289428442a014004401040184021402440404048405640604081408440"+ b"9040004120416141804185410142104248425642684200440844204480449944"+ b"124524450046014804481048404845480049584961498249454a904a00500850"+ b"1150195020508050885004514251a4519152905492540a550156545600581158"+ b"195864584059085a046010604060686000615561186260620064056410651265"+ b"84654268008002800a8041808280048118814081118201840484108415844084"+ b"608400854685948509864086608602880489118a0490109024904090a1901691"+ b"8091459200942294449451958198209902a050a085a009a100a218a450a804a9"+ )++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, qs = np.hsplit(blocks, [2])++ d = d.view(np.float16).astype(np.float32)++ qs = qs.view(np.uint32).reshape(n_blocks, -1, 2)++ db = d * (np.float32(0.5) + (qs[..., 1] >> 28).astype(np.float32)) * np.float32(0.25)+ db = db.reshape((n_blocks, -1, 1, 1))++ # get the sign indices and unpack the bits+ signs = qs[..., 1].reshape((n_blocks, -1, 1)) >> np.array([0, 7, 14, 21], dtype=np.uint32).reshape((1, 1, 4))+ ksigns = np.frombuffer(cls.ksigns, dtype=np.uint8).reshape((1, 1, 1, 128))+ signs = (signs & np.uint32(0x7F)).reshape((n_blocks, -1, 4, 1))+ signs = np.take_along_axis(ksigns, signs, axis=-1)+ signs = signs.reshape((n_blocks, -1, 4, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 1, 8))+ signs = signs & np.uint8(0x01)+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))+ signs = signs.reshape((n_blocks, -1, 4, 8))++ assert cls.grid is not None+ grid = np.take_along_axis(cls.grid, qs[..., 0].copy().view(np.uint8).reshape((n_blocks, -1, 1, 1)), axis=-2)+ grid = grid.reshape((n_blocks, -1, 4, 8))++ return (db * grid * signs).reshape((n_blocks, -1))+++class IQ2_XS(__Quant, qtype=GGMLQuantizationType.IQ2_XS):+ # iq2xs_grid, but with each byte of the original packed in 2 bits,+ # by mapping 0x08 to 0, 0x19 to 1, and 0x2b to 2.+ grid_shape = (512, 8)+ grid_map = (0x08, 0x19, 0x2b)+ grid_hex = (+ b"00000200050008000a0011001400160019002000220025002800410044004600"+ b"49005000520055005800610064008000820085008800910094009900a0000101"+ b"04010601090110011201150118011a0121012401400142014501480151015401"+ b"6001680181018401900100020202050208021102140220024102440250025502"+ b"80028a0201040404060409041004120415041804210424044004420445044804"+ b"5104540456046004810484049004000502050505080511051405200541054405"+ b"500561058005010604061006260640064206840600080208050808080a081108"+ b"14082008250841084408500858088008a008aa08010904091009400981098909"+ b"000a200a280a960aa00a01100410061009101010121015101810211024104010"+ b"4210451048105110541060106a10811084109010001102110511081111111411"+ b"2011411144115011801194119611011204120612101240126012001402140514"+ b"0814111414142014411444144914501464148014011504151015401500161416"+ b"49160118041810181218401854188618001905196619511aa91a002002200520"+ b"08200a201120142020204120442050208020a020012104211021402148216521"+ b"002222228022a82201240424102429244024002541255225992501261a26a626"+ b"002808280a28202855288828a22868299029082a202a822a882a8a2a01400440"+ b"0640094010401240154018402140244040404240454048404a40514054406040"+ b"6540814084409040004102410541084111411441204141414441504180418541"+ b"a241014204421042124229424042004402440544084411441444194420444144"+ b"4444504480449444014504451045244540459a4500460a464446504601480448"+ b"1048404845485448624800491149444950496949044a00500250055008501150"+ b"145020502850415044505050805001510451105115514051425100524452aa52"+ b"0154045410542154405460548154a154005508558055885521566856a1560058"+ b"14584158505899581a5940594259855a0160046010604060546062608660a960"+ b"006124624a62926200641664106540654565a46501686a682569066a546a626a"+ b"00800280058008801180148020802a8041804480508080808280a880aa800181"+ b"0481068110814081518159810082208280828282a082a8820184048410841284"+ b"158440846084898400854485a58518866a860088088825885a8880888288a888"+ b"0689228a808a888a968aa88a0190049010904090569084900091229164915692"+ b"89920094059444945094589429959095929541965198a6984999159a609a00a0"+ b"02a008a00aa020a02aa0a0a051a159a1a6a100a202a208a22aa280a2a0a240a4"+ b"95a465a698a60aa820a822a828a8a0a8a8a804a984a986a928aa2aaa91aaaaaa"+ )++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ qs, scales = np.hsplit(rest, [2 * QK_K // 8])++ d = d.view(np.float16).astype(np.float32)+ qs = qs.view(np.uint16)++ scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))+ scales = (scales & 0x0F).reshape((n_blocks, -1))+ db = d * (np.float32(0.5) + scales) * np.float32(0.25)+ db = db.reshape((n_blocks, -1, 1, 1))++ # get the sign indices and unpack the bits+ signs = np.frombuffer(IQ2_XXS.ksigns, dtype=np.uint8).reshape(1, 1, 128)+ signs = np.take_along_axis(signs, (qs >> 9).reshape((n_blocks, -1, 1)), axis=-1)+ signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8))+ signs = signs & np.uint8(0x01)+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))+ signs = signs.reshape((n_blocks, -1, 2, 8))++ assert cls.grid is not None+ grid = np.take_along_axis(cls.grid, (qs & np.uint16(511)).reshape((n_blocks, -1, 1, 1)), axis=-2)+ grid = grid.reshape((n_blocks, -1, 2, 8))++ return (db * grid * signs).reshape((n_blocks, -1))+++class IQ2_S(__Quant, qtype=GGMLQuantizationType.IQ2_S):+ # iq2s_grid, but with each byte of the original packed in 2 bits,+ # by mapping 0x08 to 0, 0x19 to 1, and 0x2b to 2.+ grid_shape = (1024, 8)+ grid_map = (0x08, 0x19, 0x2b)+ grid_hex = (+ b"00000200050008000a0011001400160019002000220025002800410044004600"+ b"490050005200550058006100640066006900800082008500880091009400a000"+ b"a500aa0001010401060109011001120115011801210124014001420145014801"+ b"510154015601590160016501680181018401900192019501a101a40100020202"+ b"050208021102140220022a02410244024602490250025502800285028a029402"+ b"a202010404040604090410041204150418042104240426042904400442044504"+ b"48044a0451045404560459046004620465048104840486048904900495049804"+ b"a104a40400050205050508050a05110514051605190520052505280541054405"+ b"46054905500552055505580561056405800582058505880591059405a0050106"+ b"0406060609061006150640064506480651065406600681068406900600080208"+ b"050808081108140816081908200825082a084108440846084908500852085508"+ b"580861086408800885089408aa08010904091009120915091809210940094509"+ b"480951095409600981099009000a110a140a220a280a2a0a500a990a01100410"+ b"0610091010101210151018102110241026104010421045104810511054105610"+ b"59106010621065106810811084108610901095109810a110a410001102110511"+ b"08110a1111111411161119112011221125112811411144114611491150115211"+ b"5511581161116411801182118511881191119411011204120912101215122112"+ b"2412401245125112541281128412901200140214051408141114141416141914"+ b"2014251428144114441446144914501452145514581461146414801482148514"+ b"881491149414a014011504150615091510151215151518152115241540154215"+ b"4515481551155415601581158415901500160516081611161416201641164416"+ b"50168016aa160118041806180918101815181818211840184218451848185118"+ b"541860188118841800190219051908191119141920194119441950196919a219"+ b"041a101a401a561a00200220052008201120142016201920202025202a204120"+ b"4420502052205520642080208a209420aa200121042110211221152121214021"+ b"4221452151215421602181218421902100220a22222228222a22442250228822"+ b"8a22a82201240424062409241024152418242124242440244224452448245124"+ b"5424602481248424902400250525082511251425202541254425502566258025"+ b"0126042610264026592600280528112814284128442850288a28aa2801290429"+ b"102995290a2a222a642a882a8a2a014004400640094010401240154018401a40"+ b"21402440264040404240454048404a4051405440564059406040624065408140"+ b"8440904095409840a140a4400041024105410841114114411641194120412241"+ b"2541414144414641494150415241554158416141644180418241854188419141"+ b"9441a04101420442104212421542184224424042454248425142544260428142"+ b"844200440244054408440a441144144416441944204422442544284441444444"+ b"46444944504452445544584461446444804482448544884491449444a0440145"+ b"0445064509451045124515451845214524454045424545454845514554456045"+ b"6a4581458445904500460246054608461146144620464146444650468046a546"+ b"0148044809481048124815481848214824484048424845484848514854486048"+ b"84489048004902490549084911491449204941494449504980499649014a044a"+ b"104a404a00500250055008501150145016501950205022502550285041504450"+ b"4650495050505250555058506150645080508250855088509150945001510451"+ b"0651095110511251155118512151245140514251455148515151545160518151"+ b"8451905100520552085211521452205241524452505269528052015404540654"+ b"0954105412541554185421542454405442544554485451545454605481548454"+ b"9054005502550555085511551455205541554455505580550156045610562656"+ b"405600580258055808581158145820584158445850585a588058015904591059"+ b"4059005a195a855aa85a01600460066010601260156018602160246040604560"+ b"4860516054606060846090600061026105610861116114612061416144615061"+ b"806199610462106240625662a162006405640864116414642064416444645064"+ b"806401650465106540654a656865926500669466016804681068656898680069"+ b"2a69426aa16a0080028005800880118014801980208025804180448050805280"+ b"5580588061808080858091809480018104810981108112811581188121812481"+ b"408142814581488151815481818184819081a981008205820a82118214824182"+ b"4482508201840484068409841084128415841884218440844284458448845184"+ b"5484608481848484908400850285058508851185148520854185448550858085"+ b"8a85018604861086298640860088058811881488418844885088a28801890489"+ b"40896589228a588a5a8a828aa28a019004900990109012901590189024904090"+ b"4290459048905190549060908190849090900091059111911491419144915091"+ b"5a910192049210924092a6920094029405940894119414942094419444945094"+ b"8094969401950495109540959895a19500964696649601980498109826984098"+ b"a998009949995299909a00a005a00aa014a022a02aa041a044a050a0a2a0aaa0"+ b"40a165a102a20aa222a228a22aa282a288a28aa2a8a201a404a410a440a489a4"+ b"a4a400a519a551a60aa828a8a2a854a986a908aa0aaa20aa22aa28aa88aaaaaa"+ )++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ qs, rest = np.hsplit(rest, [QK_K // 8])+ signs, rest = np.hsplit(rest, [QK_K // 8])+ qh, scales = np.hsplit(rest, [QK_K // 32])++ d = d.view(np.float16).astype(np.float32)++ scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))+ scales = (scales & 0x0F).reshape((n_blocks, -1))+ db = d * (np.float32(0.5) + scales) * np.float32(0.25)+ db = db.reshape((n_blocks, -1, 1, 1))++ # unpack the sign bits+ signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8))+ signs = signs & np.uint8(0x01)+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))+ signs = signs.reshape((n_blocks, -1, 2, 8))++ qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 1, 4))+ qs = qs.astype(np.uint16) | ((qh & 0x03).astype(np.uint16) << 8).reshape((n_blocks, -1))++ assert cls.grid is not None+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)+ grid = grid.reshape((n_blocks, -1, 2, 8))++ return (db * grid * signs).reshape((n_blocks, -1))+++class IQ3_XXS(__Quant, qtype=GGMLQuantizationType.IQ3_XXS):+ grid_shape = (256, 4)+ grid_map = (0x04, 0x0c, 0x14, 0x1c, 0x24, 0x2c, 0x34, 0x3e)+ grid_hex = (+ b"0000020004001100130017002000220031004200730075000101030110011201"+ b"2101250130013201410154017001000202020402110220022202310233023702"+ b"5102570275020103070310031203250370031304370444045704730475040105"+ b"0705320552053506640610071407160743076107011003101010121021102310"+ b"3010321034104710501000110211111120112211011203121012121221123012"+ b"7212001302132013311346136613011405145014201524154615711505162217"+ b"4017002002201120132020202220262031204220012103210521102112212121"+ b"3021632167217021002202221122172220222222372240225522012310231423"+ b"7023742335245324032527254125742501270327162745270130103012302130"+ b"2330503065307230003102312031313144314631013203321032253252327232"+ b"1133333330344734723400350635223555351436363663363337603704401740"+ b"3540374053405740744120423742404260426642074345430444514464442545"+ b"4345704505471047124730471250415070500051065126515551145232527252"+ b"0253535310542354275472540255315550562457425724604460466064602161"+ b"6161176264623063366344640565526533660367216703700570077010703270"+ b"5270267140711272457252720073157333736073217441740075027524753076"+ )++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ qs, scales = np.hsplit(rest, [QK_K // 4])++ d = d.view(np.float16).astype(np.float32)+ scales = scales.view(np.uint32)++ db = d * (np.float32(0.5) + (scales >> 28).astype(np.float32)) * np.float32(0.5)+ db = db.reshape((n_blocks, -1, 1, 1))++ # get the sign indices and unpack the bits+ signs = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 7, 14, 21], dtype=np.uint32).reshape((1, 1, 4))+ ksigns = np.frombuffer(IQ2_XXS.ksigns, dtype=np.uint8).reshape((1, 1, 1, 128))+ signs = (signs & np.uint32(0x7F)).reshape((n_blocks, -1, 4, 1))+ signs = np.take_along_axis(ksigns, signs, axis=-1)+ signs = signs.reshape((n_blocks, -1, 4, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 1, 8))+ signs = signs & np.uint8(0x01)+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))+ signs = signs.reshape((n_blocks, -1, 4, 8))++ assert cls.grid is not None+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)+ grid = grid.reshape((n_blocks, -1, 4, 8))++ return (db * grid * signs).reshape((n_blocks, -1))+++class IQ3_S(__Quant, qtype=GGMLQuantizationType.IQ3_S):+ grid_shape = (512, 4)+ grid_map = (0x01, 0x03, 0x05, 0x07, 0x09, 0x0b, 0x0d, 0x0f)+ grid_hex = (+ b"0000010002000500070010001100120014001600200021002500330040004200"+ b"4500470051005300600062007100740077000001010102010401100111011501"+ b"2001230127013101350144016101650172010002010205020702100213021602"+ b"2102250230023402420245024702510253027002730203031103150320032203"+ b"3103330336034403500352036703710375030004130417042104240432044004"+ b"4304510470040205040520052205260533054105450547056605730506061106"+ b"1306310652067106000702070407200722072607330750075407001001100210"+ b"0410101011101310151017102010221031103410361054105610611072100011"+ b"0111031106111011141121113011331141115011521170117611001212121512"+ b"1712201224123212401243125512601272120113041307131013131321132713"+ b"3013341341136213701303140514121414143114331442144614501454140115"+ b"1015131521153015321551152016241627164416461601170317101712172117"+ b"3517411762177017002001200320052007201020122014201620212023202720"+ b"3020322041204320452050205220672070207320752000210221102113211721"+ b"2221252131213421422151210122042207222122232230223722412253225722"+ b"7122742200230223052311232223242331233323422350236623012407242024"+ b"2324322435244124722475240425112522253725402553257025002602260726"+ b"2126552661260527112726273027432750270230113013301530173022303130"+ b"3330353042304430473051306330713001310331053114312131233140316031"+ b"7231763100321232203232323432503201331033143321332333273330334133"+ b"4333473355337333033411341634223431345234603464340135103512352535"+ b"3235443556357335163641360137033720372237353700400440124020402440"+ b"2740324041405040704002410741114113412241304135414341514155410142"+ b"0342104215422142334240425742624270420443114313432043224331433543"+ b"0044024424443744404471440545074521456245134634466046104715473047"+ b"4347514702501050145022504050445047505250665074500151035105511251"+ b"2151325172510052115223523052365253520253075310532753445351536553"+ b"7353015404542054325446541255265551555355425602570457225711601360"+ b"1560316033606060006120612761646112623462426255626262706200631463"+ b"2163406325644364626400650365346560650566406611671367007004700770"+ b"2070227036704070547062700271117124714371457101720472107216722172"+ b"3072517202733273357353730174057413742074507422754275027631760077"+ )++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ qs, rest = np.hsplit(rest, [QK_K // 4])+ qh, rest = np.hsplit(rest, [QK_K // 32])+ signs, scales = np.hsplit(rest, [QK_K // 8])++ d = d.view(np.float16).astype(np.float32)++ scales = scales.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))+ scales = (scales & 0x0F).reshape((n_blocks, -1))+ db = d * (1 + 2 * scales)+ db = db.reshape((n_blocks, -1, 1, 1))++ # unpack the sign bits+ signs = signs.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8)], dtype=np.uint8).reshape((1, 1, 8))+ signs = signs & np.uint8(0x01)+ signs = np.where(signs == 0, np.float32(1), np.float32(-1))+ signs = signs.reshape((n_blocks, -1, 4, 8))++ qh = qh.reshape((n_blocks, -1, 1)) >> np.array([i for i in range(8)], dtype=np.uint8)+ qh = (qh & 0x01).astype(np.uint16).reshape((n_blocks, -1))+ qs = qs.astype(np.uint16) | (qh << 8)++ assert cls.grid is not None+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)+ grid = grid.reshape((n_blocks, -1, 4, 8))++ return (db * grid * signs).reshape((n_blocks, -1))+++class IQ1_S(__Quant, qtype=GGMLQuantizationType.IQ1_S):+ # iq1s_grid, with each byte packed into 2 bits+ # -1, 0, 1 <=> 0, 1, 2+ grid_shape = (2048, 8)+ grid_map = (-1, 0, 1)+ grid_hex = (+ b"00000200050008000a00110015002000220028002a0045005100540056006500"+ b"8000820088008a009500a000a200a800aa000401050111011401160119011a01"+ b"2501410146014901520155015a0161016401660168018501910194019601a501"+ b"0002020208020a0215022002220228022a024502510259026402690280028202"+ b"88028a02910295029902a002a202a802aa021104140416042504410449045504"+ b"5a046404650491049904a5040105040505050605150518051a05290540054505"+ b"4a0550055105540555055605590560056205650568056a058105910595059805"+ b"9a05a105a405a505a605a9051406190641064406500652065506580660066106"+ b"6606690685069106940699060008020808080a0815082008220828082a084508"+ b"5108560865088008820888088a089508a008a208a808aa080509110914091909"+ b"2409250941095009510955096109640969099109940996099909a509000a020a"+ b"080a0a0a150a200a220a280a2a0a450a510a590a610a650a800a820a850a880a"+ b"8a0a950aa00aa20aa80aaa0a1010111014101910241025104110441050105510"+ b"58106110641065106910911094109610a110a510011104110611091110111211"+ b"1511181121112411291145114a11501151115211541155115611591160116511"+ b"841192119511a111a41111121412161225124012461249125212551258125a12"+ b"641266128512911294129612a512011406140914141415141814191421142614"+ b"41144514461448144a1451145414551456145914621465146814841489149014"+ b"94149514981499149a14a114a414a514a914021505150a151115141515151615"+ b"191520152215251528152a154115441545154615511552155415551556155915"+ b"5a1561156415651566156915801582158415851588158a159015911594159515"+ b"961599159a15a015a215a51501160416051606161516161618161a1621162616"+ b"401642164416451648164a165116551656165816591661166416651668166916"+ b"6a1686168a1692169516a416a916111816182518411844184618491850185518"+ b"58185a1860186118641866186918851891189418a5181019121915191a192119"+ b"25194219441945194819511954195519561959195a19601965196a1989199119"+ b"921995199819a119a619a919091a161a241a261a441a461a491a501a521a551a"+ b"581a611a661a691a851a911a961a9a1a0020022008200a201520202022202520"+ b"28202a20452051205920612065208020822088208a209520a020a220a520a820"+ b"aa2005211121142119212521422144214921552158215a216121642165216621"+ b"8521902196219921a521012208220a22112215222022222228222a2245225122"+ b"562259226522812288228a2291229522a022a222a822aa220524142416241924"+ b"252444244524462449245224552458245a2466248524912494249924a124a524"+ b"0925152521252925402545254825512554255525592562256525682589259025"+ b"9425952598259a25a125a425a625a92505261026122619262526412649265526"+ b"6026612669268426862690269a260028022808280a2815282028222828282a28"+ b"45285128542865288028822888288a28a028a228a828aa280929112914291929"+ b"2529462949295229552961296429662969298529902996299929a429a529002a"+ b"022a082a0a2a202a222a282a2a2a452a512a562a592a652a802a822a882a8a2a"+ b"952aa02aa22aa82aaa2a054011401640254049405240554058405a4061406440"+ b"664094409940a140a6400041014104410641094112411541164118411a412141"+ b"26412941454148414a41514154415541564159415a41654168416a4181418441"+ b"8641904192419541a041a141a241054211421442164225424142524255425a42"+ b"6442694289429442a5420144154419442944454448444a445144544455445644"+ b"61446244654468446a44814486448944904492449544a044a144a94401450245"+ b"05450a4511451445154516451945204525452a45414544454545464549455045"+ b"5145544555455645584559456145644565456645694582458445854588459145"+ b"94459545964599459a45a545a845aa450146054609461446154618461a462146"+ b"2446294640464246454648465046514652465546564659466246654668468146"+ b"85468a4694469546a146a446a6460548114815481a4825484248494850485548"+ b"5848614864486648694885489148944896489948a5480149054906490a491049"+ b"144915491849214924492649404945494a495149524954495549564959496049"+ b"6249654966496a49864989499249954996499849a149a449a649a949164a444a"+ b"464a494a554a584a5a4a644a694a944aa54a0150045005500650095012501550"+ b"1a50215024502950405045504850515054505550565059506550685086508950"+ b"95509850a050a150a650a9500551085109510a51115114511551165118511951"+ b"20512551265128512a5141514451455146514951505151515251545155515651"+ b"585159515a51615164516551665169518251855191519451955196519951a051"+ b"a551aa5101520652125215521a5221522452425245524a525152545255525652"+ b"595262526552855290529252955299529a52a452045405541154145415541654"+ b"185419542154255428542a54415444544554465449544a545054515454545554"+ b"5654585459545a54615462546454655466546954805488548a54915494549554"+ b"96549954a154a454a554aa540155025504550555065509551055115512551455"+ b"1555165519551a55215524552555265529554055415542554455455546554855"+ b"4955505551555255545555555655585559555a55605561556455655566556855"+ b"69556a5581558455855589558a559055915594559555965598559955a155a455"+ b"a555a655a9550056015602560456065608560956115614561556185619562056"+ b"2156225624562556265628562956415645564656485649564a56505651565256"+ b"545655565656585659565a566156645665566956825685568656885689568a56"+ b"915695569a56a256a556a656a856a95604580558065809581058155818582158"+ b"2a58455848584a58515854585558565858585958605862586458655882588958"+ b"9058925895589858a158a9580159025905590a59115914591559165919592559"+ b"41594459455946594959505951595259545955595659585959595a5961596459"+ b"655966596959815985598959915994599559965998599959a559045a085a155a"+ b"1a5a205a255a265a295a455a485a495a515a555a565a585a595a625a655a685a"+ b"6a5a815a8a5a925a955a965a985a9a5aa15a0560146016601960256044605060"+ b"5560566058605a60616064606660696081609660a56001610461066109611261"+ b"15612161226126612961456149615161556156615961656166616a6184618a61"+ b"92619561a161a661a96111621662196240624162466255625662586260628562"+ b"91629662a56211641264156416641a6421642664296440644264456448644a64"+ b"516454645564566459645a646064626465648464856489649064926494649564"+ b"966498649a64a164a464a964056508650a651165156516651965446545654665"+ b"496550655165546555655665596561656465656566656965866589658a659165"+ b"9565966599659a65a265a565a665a86502660966156620662666286629664066"+ b"456648664a66516654665566566658665a666066656668668066826685668a66"+ b"9466966698669966a066a466a666aa661668196825684168526855685a686168"+ b"6968856891689868a66801690469106915692169246926692969406941694569"+ b"4669486951695469556956695969606965696a69826984698a699569a169a469"+ b"a569a969116a166a186a416a446a496a506a556a586a5a6a646a656a696a866a"+ b"946a986a9a6aa66a0080028008800a802080228028802a804580508051805480"+ b"5680598065808080828088808a809580a080a280a880aa800581118114811681"+ b"1981258141814481498150815281558156815881598164816681698185818981"+ b"948196819981a5810082028208820a8215822082228228822a82518254825982"+ b"65828082828288828a829582a082a282a882aa82148419844184448451845584"+ b"5a846184648469849484998401850985128515851a8526852985408541854585"+ b"4885518554855585568559855a856585668568856a8581858485868589859085"+ b"928595859885a68511861686198625864186448649864a865086558659865a86"+ b"618666866a86858691869a86a4860088028808880a8815882088228828882a88"+ b"41884588518854885988658869888088828888888a889588a088a288a888aa88"+ b"05890689118914891689258941894489468949895089528955895a8961896489"+ b"858996899989a589008a028a088a0a8a158a208a228a288a2a8a458a518a548a"+ b"568a808a828a888a8a8a958aa08aa28aa88aaa8a059011901690189019902590"+ b"419046904990559058905a9069906a9085909190949096909990a59001910491"+ b"069109911091159118911a912191249126912991409145915091519154915591"+ b"569159916291659184918691929195919891a191a491a691a991059211921492"+ b"19922592449246924992509252925592589266926992859294929692a9920194"+ b"04940694109415941894269440944a9451945494559456945894599460946194"+ b"62946594849486949294949495949894a194a9940095059508950a9510951195"+ b"14951595169519952195259529952a9541954495459546954995509551955295"+ b"549555955695589559955a956195649565956695699581958595889591959295"+ b"94959595969599959a95a095a295a595a895aa95019604961096159619962096"+ b"2696299645964896499651965296559656965996659668968296849689968a96"+ b"929694969596a496a696a9960598169819982598419846985098529855985698"+ b"5a98649865988598919896989998a59804990699099910991299159918991a99"+ b"209921992499269940994299459948994a995199549955995699599962996599"+ b"66996a99819984999099929995999a99a199a699059a159a259a449a469a499a"+ b"509a559a589a619a859a919a949a959a969a00a002a008a00aa015a020a022a0"+ b"28a02aa045a051a054a056a059a080a082a088a08aa095a0a0a0a2a0a8a0aaa0"+ b"05a109a111a114a116a119a11aa146a149a151a155a158a15aa161a164a185a1"+ b"90a192a196a199a102a208a20aa210a219a222a228a22aa245a251a256a259a2"+ b"65a280a282a288a28aa295a2a0a2a2a2a8a2aaa219a425a441a444a450a454a4"+ b"55a458a45aa461a465a466a468a469a485a406a509a510a512a515a518a526a5"+ b"29a542a545a551a554a555a556a559a565a56aa581a584a585a586a589a592a5"+ b"95a598a505a611a616a61aa621a625a644a646a64aa652a655a656a658a660a6"+ b"62a686a690a695a696a699a6a1a6a4a6a6a600a802a808a80aa820a822a828a8"+ b"2aa851a854a856a859a880a882a888a88aa895a8a0a8a2a8a8a8aaa805a914a9"+ b"19a921a925a941a950a955a95aa961a966a969a990a996a900aa02aa08aa0aaa"+ b"20aa22aa28aa2aaa51aa54aa56aa80aa82aa88aa8aaa95aaa0aaa2aaa8aaaaaa"+ )++ delta = np.float32(0.125)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ qs, qh = np.hsplit(rest, [QK_K // 8])++ d = d.view(np.float16).astype(np.float32)+ qh = qh.view(np.uint16)++ dl = d * (2 * ((qh >> 12) & 7) + 1)+ dl = dl.reshape((n_blocks, -1, 1, 1))+ delta = np.where((qh & np.uint16(0x8000)) == 0, cls.delta, -cls.delta)+ delta = delta.reshape((n_blocks, -1, 1, 1))++ qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 3, 6, 9], dtype=np.uint16).reshape((1, 1, 4))+ qs = qs.astype(np.uint16) | ((qh & 7) << 8).reshape((n_blocks, -1))++ assert cls.grid is not None+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)+ grid = grid.reshape((n_blocks, -1, 4, 8))++ return (dl * (grid + delta)).reshape((n_blocks, -1))+++class IQ1_M(__Quant, qtype=GGMLQuantizationType.IQ1_M):+ grid_shape = IQ1_S.grid_shape+ grid_map = IQ1_S.grid_map+ grid_hex = IQ1_S.grid_hex++ delta = IQ1_S.delta++ # Okay *this* type is weird. It's the only one which stores the f16 scales in multiple parts.+ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ qs, rest = np.hsplit(blocks, [QK_K // 8])+ qh, scales = np.hsplit(rest, [QK_K // 16])++ # The f16 scale is packed across multiple bytes+ scales = scales.view(np.uint16)+ d = (scales.reshape((n_blocks, 4)) & np.uint16(0xF000)) >> np.array([12, 8, 4, 0], dtype=np.uint16).reshape((1, 4))+ d = d[..., 0] | d[..., 1] | d[..., 2] | d[..., 3]+ d = d.view(np.float16).astype(np.float32).reshape((n_blocks, 1))++ scales = scales.reshape(n_blocks, -1, 1) >> np.array([0, 3, 6, 9], dtype=np.uint16).reshape((1, 1, 4))+ scales = (scales & 0x07).reshape((n_blocks, -1))+ dl = d * (2 * scales + 1)+ dl = dl.reshape((n_blocks, -1, 2, 1, 1))++ qh = qh.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))+ qs = qs.astype(np.uint16) | ((qh & 0x07).astype(np.uint16) << 8).reshape((n_blocks, -1))++ delta = np.where(qh & 0x08 == 0, cls.delta, -cls.delta)+ delta = delta.reshape((n_blocks, -1, 2, 2, 1))++ assert cls.grid is not None+ grid = np.take_along_axis(cls.grid, qs.reshape((n_blocks, -1, 1, 1)), axis=-2)+ grid = grid.reshape((n_blocks, -1, 2, 2, 8))++ return (dl * (grid + delta)).reshape((n_blocks, -1))+++class IQ4_NL(__Quant, qtype=GGMLQuantizationType.IQ4_NL):+ kvalues = (-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113)++ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, qs = np.hsplit(blocks, [2])++ d = d.view(np.float16).astype(np.float32)++ qs = qs.reshape((n_blocks, -1, 1, cls.block_size // 2)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))++ qs = (qs & np.uint8(0x0F)).reshape((n_blocks, -1, 1))++ kvalues = np.array(cls.kvalues, dtype=np.int8).reshape(1, 1, 16)+ qs = np.take_along_axis(kvalues, qs, axis=-1).astype(np.float32).reshape((n_blocks, -1))++ return (d * qs)+++class IQ4_XS(__Quant, qtype=GGMLQuantizationType.IQ4_XS):+ @classmethod+ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray:+ n_blocks = blocks.shape[0]++ d, rest = np.hsplit(blocks, [2])+ scales_h, rest = np.hsplit(rest, [2])+ scales_l, qs = np.hsplit(rest, [QK_K // 64])++ d = d.view(np.float16).astype(np.float32)+ scales_h = scales_h.view(np.uint16)++ scales_l = scales_l.reshape((n_blocks, -1, 1)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2))+ scales_h = scales_h.reshape((n_blocks, 1, -1)) >> np.array([2 * i for i in range(QK_K // 32)], dtype=np.uint16).reshape((1, -1, 1))+ scales_l = scales_l.reshape((n_blocks, -1)) & np.uint8(0x0F)+ scales_h = scales_h.reshape((n_blocks, -1)).astype(np.uint8) & np.uint8(0x03)++ scales = (scales_l | (scales_h << np.uint8(4))).astype(np.int8) - np.int8(32)+ dl = (d * scales.astype(np.float32)).reshape((n_blocks, -1, 1))++ qs = qs.reshape((n_blocks, -1, 1, 16)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 1, 2, 1))+ qs = qs.reshape((n_blocks, -1, 32, 1)) & np.uint8(0x0F)++ kvalues = np.array(IQ4_NL.kvalues, dtype=np.int8).reshape((1, 1, 1, -1))+ qs = np.take_along_axis(kvalues, qs, axis=-1).astype(np.float32).reshape((n_blocks, -1, 32))++ return (dl * qs).reshape((n_blocks, -1))diff --git a/tests/kernels/test_ggml.py b/tests/kernels/test_ggml.pynew file mode 100644index 00000000..ea6a5d68--- /dev/null+++ b/tests/kernels/test_ggml.py@@ -0,0 +1,84 @@+"""Packed GGUF ops against gguf-py's CPU dequantization and PyTorch math."""++import numpy as np+import pytest+import torch+import torch.nn.functional as F++from freetoken.kernel import ggml+from freetoken.third_party.gguf import GGMLQuantizationType as QType+from freetoken.third_party.gguf import quants++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a GPU")+DTYPES = [torch.float32, torch.bfloat16, torch.float16]+++def _q4_weights(shape, seed=690):+ values = np.random.default_rng(seed).normal(0, 0.1, shape).astype(np.float32)+ packed = quants.quantize(values, QType.Q4_0)+ reference = torch.from_numpy(quants.dequantize(packed, QType.Q4_0))+ # ggml's last MMQ tile may read an extra partial row.+ storage = torch.zeros(*packed.shape[:-2], packed.shape[-2] + 1, packed.shape[-1], dtype=torch.uint8, device="cuda")+ storage[..., :-1, :].copy_(torch.from_numpy(packed))+ return storage[..., :-1, :], reference+++def _activation(shape, dtype, seed=19):+ rng = np.random.default_rng(seed)+ half = rng.integers(-127, 128, (*shape[:-1], shape[-1] // 32, 16))+ half[..., 0] = 127+ # Exactly representable q8_1 inputs with zero block sums isolate weight math.+ values = np.concatenate([half, -half], axis=-1).reshape(shape).astype(np.float32) / 64+ return torch.from_numpy(values).to("cuda", dtype)++[email protected]("dtype", DTYPES)+def test_q4_dequantize_and_rows(dtype):+ w, ref = _q4_weights((64, 256))+ actual = ggml.dequantize(w, int(QType.Q4_0), 256, dtype)+ torch.testing.assert_close(actual.cpu(), ref.to(dtype), atol=0, rtol=0)+ ids = torch.tensor([63, 0, 17, 17], dtype=torch.int32, device="cuda")+ rows = ggml.get_rows(w, int(QType.Q4_0), 256, ids, dtype)+ torch.testing.assert_close(rows.cpu(), ref[ids.cpu().long()].to(dtype), atol=0, rtol=0)++[email protected]("dtype", DTYPES)[email protected]("batch", [1, 17])+def test_q4_matrix_matches_cpu(dtype, batch):+ w, ref = _q4_weights((64, 256))+ x = _activation((batch, 256), dtype)+ actual = ggml.mul_mat(w, int(QType.Q4_0), x)+ expected = F.linear(x.cpu().float(), ref).to(dtype)+ torch.testing.assert_close(actual.cpu(), expected, atol=2e-4, rtol=1e-3)++[email protected]("dtype", DTYPES)[email protected]("batch", [1, 17])+def test_q4_moe_routes_match_cpu(dtype, batch):+ w, ref = _q4_weights((4, 64, 256))+ x = _activation((batch, 256), dtype)+ ids = torch.tensor([[i % 4, (i + 1) % 4] for i in range(batch)], dtype=torch.int32, device="cuda")+ actual = ggml.mul_mat_id(w, int(QType.Q4_0), x, ids)+ expected = torch.einsum("tk, tjok -> tjo", x.cpu().float(), ref[ids.cpu().long()]).to(dtype)+ torch.testing.assert_close(actual.cpu(), expected, atol=2e-4, rtol=1e-3)++[email protected]("dtype", DTYPES)+def test_q4_shared_q8_matches_internal_quantization(dtype):+ w, _ = _q4_weights((64, 256))+ x = _activation((1, 256), dtype)+ expected = ggml.mul_mat(w, int(QType.Q4_0), x)+ actual = ggml.mul_mat(w, int(QType.Q4_0), x, xq=ggml.quantize_q8_1(x))+ torch.testing.assert_close(actual, expected, atol=0, rtol=0)+++def test_q4_nondefault_stream():+ w, ref = _q4_weights((64, 256))+ torch.cuda.synchronize()+ stream = torch.cuda.Stream()+ with torch.cuda.stream(stream):+ x = _activation((1, 256), torch.float32)+ actual = ggml.mul_mat(w, int(QType.Q4_0), x)+ copied = actual.clone()+ stream.synchronize()+ torch.testing.assert_close(copied.cpu(), F.linear(x.cpu(), ref), atol=2e-4, rtol=1e-3)diff --git a/tests/models/deepseek_v41/harness.py b/tests/models/deepseek_v41/harness.pyindex 98b19e2d..83c0ec61 100644--- a/tests/models/deepseek_v41/harness.py+++ b/tests/models/deepseek_v41/harness.py@@ -62,7 +62,7 @@ class TinyEngine:from freetoken.moe.offload_cache import OffloadMoeCache, attach_offload_moe_cacheself.banks = load_expert_banks(- checkpoint, mc, method=method, device=self.device, dtype=torch.bfloat16, parallel=False,+ checkpoint, mc, method=method, device=self.device, parallel=False,)self.expert_cache = OffloadMoeCache(num_layers=mc.num_moe_layers, num_experts=mc.num_experts,@@ -76,7 +76,7 @@ class TinyEngine:from freetoken.moe.expert_banks import attach_resident_banksattach_resident_banks(list(iter_moe_layers(self.model)), load_expert_banks(- checkpoint, mc, method=method, device=self.device, dtype=torch.bfloat16, parallel=False, resident=True,+ checkpoint, mc, method=method, device=self.device, parallel=False, resident=True,))for layer in self.model.engram_layers():layer.attach_table(engram_table if engram_table is not None else ZeroEngramTable(layer.width, self.device))diff --git a/tests/moe/test_cpu_moe_q4_0.py b/tests/moe/test_cpu_moe_q4_0.pyindex 7babb5d4..9452f95c 100644--- a/tests/moe/test_cpu_moe_q4_0.py+++ b/tests/moe/test_cpu_moe_q4_0.py@@ -3,7 +3,7 @@ the gemma4 GGUF checkpoint).The CPU W4A16 GEMV (``q4_0_dot`` in csrc/cpu_moe/kernels/q4_0.cpp) reads the *same*packed Q4_0 banks the GPU offload path streams and dequantizes weights inside the-K-loop. We check it against the reference dequant (models/gguf/dequant.py) + the+K-loop. We check it against the reference dequant (gguf-py's numpy dequantize) + theproduction bf16 GPU decode kernel on byte-identical banks: both are W4A16, so theonly spread is weight bf16-rounding + reduction order -> tight relative tolerance.@@ -39,11 +39,11 @@ def _pack_q4_0(nibbles: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:def _dequant_bank(packed: torch.Tensor, K: int, dev) -> torch.Tensor:"""[S, OUT, K//32*18] packed Q4_0 -> [S, OUT, K] bf16 (storage order == elem order)."""- from freetoken.models.gguf.dequant import GGML_Q4_0, dequantize+ from freetoken.third_party.gguf import GGMLQuantizationType, quantsS, OUT, _ = packed.shape- flat = dequantize(packed.reshape(-1), GGML_Q4_0, torch.bfloat16)- return flat.reshape(S, OUT, K).to(dev)+ values = quants.dequantize(packed.numpy(), GGMLQuantizationType.Q4_0)+ return torch.from_numpy(values).reshape(S, OUT, K).to(dev, torch.bfloat16)def _make_q4_0_cache(L, E, H, I, seed=0):@@ -115,8 +115,10 @@ def test_cpu_decode_q4_0_matches_dequant_then_gpu(bs):def test_cpu_decode_q4_0_matches_ggml_mmvq():"""Sanity: the CPU W4A16 GEMV lands close to the GPU ggml MMVQ (W4A8) kernel theoffload path uses -- a looser tol since MMVQ quantizes activations to int8."""+ from freetoken.kernel import ggml+ from freetoken.layers.activation import gated_act_and_mulfrom freetoken.moe.cpu_executor import CpuMoeExecutor- from freetoken.moe.fused_q4_0 import fused_experts_gguf_q4_0+ from freetoken.third_party.gguf import GGMLQuantizationTypetorch.manual_seed(77)L, E, H, I, top_k = 2, 16, 2816, 704, 8@@ -137,11 +139,12 @@ def test_cpu_decode_q4_0_matches_ggml_mmvq():torch.cuda.synchronize()b = cache.bank_sources- gate_up_layer = b["gate_up"][layer].to(dev)- down_layer = b["down"][layer].to(dev)- gpu_out = fused_experts_gguf_q4_0(- hidden, gate_up_layer, down_layer, w, ids.clone(), "gelu_tanh"- ).float()+ q4_0 = int(GGMLQuantizationType.Q4_0)+ h = ggml.mul_mat_id(b["gate_up"][layer].to(dev), q4_0, hidden.float(), ids)+ inter = torch.empty(bs, top_k, I, device=dev)+ gated_act_and_mul("gelu_tanh", h.view(-1, 2 * I), inter.view(-1, I))+ y = ggml.mul_mat_id(b["down"][layer].to(dev), q4_0, inter, ids)+ gpu_out = ggml.moe_weighted_sum(y, w)rel = (cpu_out - gpu_out).abs().max() / (gpu_out.abs().max() + 1e-6)assert rel < 6e-2, f"q4_0 vs ggml-mmvq rel err {rel.item()}"diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.pyindex d3c8d9d8..a5a7adb9 100644--- a/tests/moe/test_offload.py+++ b/tests/moe/test_offload.py@@ -681,7 +681,6 @@ def test_slot_pools_of_two_bank_layouts_keep_their_rows_through_prefill():check_resident()decode(20)- cache.zero_inactive_route_slots = Truecache.begin_prefill()for layer in range(L):cache.prefetch_prefill_layer(layer)@@ -735,7 +734,6 @@ def test_slot_pools_split_layers_of_one_bank_layout_by_quant_tag():cache.set_bank_sources(sources, layer_tags=tags)assert cache.bank_caches["down"].shape == (20, 32, 144)# the staging buffers cover pool 0 and slots [0, 6) of pool 1, so both slot 0 rows end up with another quant's bytes- cache.zero_inactive_route_slots = Truecache.begin_prefill()for layer in range(L):cache.prefetch_prefill_layer(layer)@@ -748,6 +746,56 @@ def test_slot_pools_split_layers_of_one_bank_layout_by_quant_tag():assert not view[0].any()[email protected](not torch.cuda.is_available(), reason="needs CUDA")[email protected]("wide", [(), (1, 3)], ids=["cpu-mode", "two-layouts"])+def test_slot_pools_and_staging_buffers_end_in_zero_mmq_tails(wide):+ from freetoken.layers.quantization.ggml_types import mmq_tail_bytes, row_bytes+ from freetoken.moe.offload_cache import OffloadMoeCache+ from freetoken.third_party.gguf import GGMLQuantizationType++ _init_tp()+ torch.manual_seed(0)+ L, E, top_k = 4, 8, 4++ def layout(ggml_type):+ """(rows, row bytes, tail bytes) of gate_up and down for Gemma-4 26B-A4B's K = 2816 and 704, with fewer rows."""+ return tuple((rows, row_bytes(ggml_type, k), mmq_tail_bytes(ggml_type, k)) for rows, k in ((32, 2816), (64, 704)))++ q4_0, q8_0 = layout(GGMLQuantizationType.Q4_0), layout(GGMLQuantizationType.Q8_0)+ shapes = [q8_0 if layer in wide else q4_0 for layer in range(L)]+ roles = ("gate_up", "down")+ sources = {role: [torch.randint(1, 255, (E, *shapes[l][k][:2]), dtype=torch.uint8).pin_memory() for l in range(L)] for k, role in enumerate(roles)}+ tails = {role: [shapes[l][k][2] for l in range(L)] for k, role in enumerate(roles)}+ # cpu mode: the slots are exactly the two staging buffers+ pools = dict(pool_of_layer=[2, 0, 3, 1], pool_sizes=[6, 5, 12, 10], prefill_hit_d2d=True) if wide else dict(pool_of_layer=[0] * L, pool_sizes=[2 * E])+ cache = OffloadMoeCache(num_layers=L, num_experts=E, cache_size=sum(pools["pool_sizes"]), device=torch.device("cuda"), prefill_overlap=True, **pools)+ cache.set_bank_sources(sources, row_tail_bytes=tails)+ assert cache.prefill_overlap++ def assert_zero_tails(layer: int, views) -> None:+ torch.cuda.synchronize()+ for k, ((_, bank), view) in enumerate(zip(cache.banks, views)):+ end, tail = view.storage_offset() + view.numel(), shapes[layer][k][2]+ assert end + tail <= bank.numel() and not bank[end : end + tail].any(), (layer, k)++ for _ in range(20):+ for layer in range(L):+ cache.ensure_experts(layer, torch.randperm(E)[:top_k].to(torch.int32).view(1, top_k).cuda())+ cache.copy_missing()+ cache.begin_prefill()+ for layer in range(L):+ cache.prefetch_prefill_layer(layer)+ cache.prefetch_prefill_layer(layer + 1)+ assert_zero_tails(layer, cache.wait_prefill_layer(layer))+ cache.release_prefill_layer(layer)+ for layer in range(L):+ assert_zero_tails(layer, cache.pool_bank_views(layer))+ if wide:+ cache.materialize_layer(0)+ cache.copy_missing()+ assert_zero_tails(0, cache.stage_views(0, 0))++@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")def test_offload_cache_rebuild_frees_the_old_slots_first():from freetoken.moe.offload_cache import OffloadMoeCache